This is an automated email from the ASF dual-hosted git repository. abhay pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push: new 774d159e2 RANGER-4185: Improve debugging messages when policy-deltas are enabled 774d159e2 is described below commit 774d159e2a2967132e8a1eda7f5ddeed08b37a55 Author: Abhay Kulkarni <ab...@apache.org> AuthorDate: Tue Apr 18 17:15:15 2023 -0700 RANGER-4185: Improve debugging messages when policy-deltas are enabled --- .../ranger/plugin/model/RangerPolicyDelta.java | 2 +- .../ranger/plugin/policyengine/PolicyEngine.java | 10 +++++++- .../ranger/plugin/util/RangerPolicyDeltaUtil.java | 2 +- .../java/org/apache/ranger/biz/ServiceDBStore.java | 29 +++++++++++++++++++--- .../java/org/apache/ranger/biz/TagDBStore.java | 3 +++ .../ranger/common/RangerServicePoliciesCache.java | 2 +- .../RangerTransactionSynchronizationAdapter.java | 15 +++++++++-- 7 files changed, 53 insertions(+), 10 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java index 33183727c..e4d9b3a40 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPolicyDelta.java @@ -87,7 +87,7 @@ public class RangerPolicyDelta implements java.io.Serializable { public void setId(Long id) { this.id = id;} - private void setChangeType(Integer changeType) { this.changeType = changeType; } + public void setChangeType(Integer changeType) { this.changeType = changeType; } private void setPoliciesVersion(Long policiesVersion) { this.policiesVersion = policiesVersion; } diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java index 3864f30d2..86b6cd376 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/PolicyEngine.java @@ -200,7 +200,15 @@ public class PolicyEngine { this.pluginContext = pluginContext; this.lock = new RangerReadWriteLock(isUseReadWriteLock); - LOG.info("Policy engine will" + (isUseReadWriteLock ? " " : " not ") + "perform in place update while processing policy-deltas."); + Boolean hasPolicyDeltas = RangerPolicyDeltaUtil.hasPolicyDeltas(servicePolicies); + + if (hasPolicyDeltas != null) { + if (hasPolicyDeltas.equals(Boolean.TRUE)) { + LOG.info("Policy engine will" + (isUseReadWriteLock ? " " : " not ") + "perform in place update while processing policy-deltas."); + } else { + LOG.info("Policy engine will" + (isUseReadWriteLock ? " " : " not ") + "perform in place update while processing policies."); + } + } this.pluginContext.setAuthContext(new RangerAuthContext(null, roles)); diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java index 86b18aace..b47888e9a 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPolicyDeltaUtil.java @@ -42,7 +42,7 @@ public class RangerPolicyDeltaUtil { public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<RangerPolicyDelta> deltas, String serviceType) { if (LOG.isDebugEnabled()) { - LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); + LOG.debug("==> applyDeltas(serviceType=" + serviceType + ", deltas=" + deltas + ")"); } List<RangerPolicy> ret; diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index e52a92e04..60903cc97 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -393,6 +393,12 @@ public class ServiceDBStore extends AbstractServiceStore { isRolesDownloadedByService = config.getBoolean("ranger.support.for.service.specific.role.download", false); SUPPORTS_IN_PLACE_POLICY_UPDATES = SUPPORTS_POLICY_DELTAS && config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES, RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT); + LOG.info("SUPPORTS_POLICY_DELTAS=" + SUPPORTS_POLICY_DELTAS); + LOG.info("RETENTION_PERIOD_IN_DAYS=" + RETENTION_PERIOD_IN_DAYS); + LOG.info("TAG_RETENTION_PERIOD_IN_DAYS=" + TAG_RETENTION_PERIOD_IN_DAYS); + LOG.info("isRolesDownloadedByService=" + isRolesDownloadedByService); + LOG.info("SUPPORTS_IN_PLACE_POLICY_UPDATES=" + SUPPORTS_IN_PLACE_POLICY_UPDATES); + TransactionTemplate txTemplate = new TransactionTemplate(txManager); final ServiceDBStore dbStore = this; @@ -2924,11 +2930,16 @@ public class ServiceDBStore extends AbstractServiceStore { @Override public ServicePolicies getServicePolicyDeltas(String serviceName, Long lastKnownVersion) throws Exception { - boolean getOnlyDeltas = true; - if (LOG.isDebugEnabled()) { - LOG.debug("Support for incremental policy updates enabled using \"ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA + "\" configuation parameter :[" + SUPPORTS_POLICY_DELTAS +"]"); + ServicePolicies ret = null; + + if (SUPPORTS_POLICY_DELTAS) { + if (LOG.isDebugEnabled()) { + LOG.debug("Support for incremental policy updates enabled using \"ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_POLICY_DELTA + "\" configuation parameter :[" + SUPPORTS_POLICY_DELTAS + "]"); + } + ret = getServicePolicies(serviceName, lastKnownVersion, true, SUPPORTS_POLICY_DELTAS); } - return getServicePolicies(serviceName, lastKnownVersion, getOnlyDeltas, SUPPORTS_POLICY_DELTAS); + + return ret; } @Override @@ -3104,6 +3115,9 @@ public class ServiceDBStore extends AbstractServiceStore { break; } } + policyDeltasForPolicy.clear(); + policyDeltas.get(index).setChangeType(RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); + policyDeltasForPolicy.add(policyDeltas.get(index)); index++; break; case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: @@ -3174,8 +3188,15 @@ public class ServiceDBStore extends AbstractServiceStore { break; } if (policyDeltasForPolicy != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Processed multiple deltas for policy:[" + entry.getKey() + "], compressed-deltas:[" + policyDeltasForPolicy + "]"); + } + if (policyDeltasForPolicy.size() > 1) { + LOG.error("More than one Compressed-deltas for policy:[" + entry.getKey() + "], compressed-deltas:[" + policyDeltasForPolicy + "].. Should not have come here!!"); + } ret.addAll(policyDeltasForPolicy); } else { + LOG.error("Error processing deltas for policy:[" + entry.getKey() + "], Cannot compress deltas"); ret = null; break; } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java index e434cf1bb..fb912d4f8 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java @@ -1367,6 +1367,9 @@ public class TagDBStore extends AbstractTagStore { SUPPORTS_TAG_DELTAS = config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_TAG_DELTA, RangerCommonConstants.RANGER_ADMIN_SUFFIX_TAG_DELTA_DEFAULT); SUPPORTS_IN_PLACE_TAG_UPDATES = SUPPORTS_TAG_DELTAS && config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES, RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_TAG_UPDATES_DEFAULT); IS_SUPPORTS_TAG_DELTAS_INITIALIZED = true; + + LOG.info("SUPPORTS_TAG_DELTAS=" + SUPPORTS_TAG_DELTAS); + LOG.info("SUPPORTS_IN_PLACE_TAG_UPDATES=" + SUPPORTS_IN_PLACE_TAG_UPDATES); } } diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java index a34d7d1d7..21f06834d 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerServicePoliciesCache.java @@ -351,12 +351,12 @@ public class RangerServicePoliciesCache { if (LOG.isDebugEnabled()) { LOG.debug("Retrieved policy-deltas from database. These will be applied on top of ServicePolicy version:[" + cachedServicePoliciesVersion +"], policy-deltas:[" + servicePoliciesFromDb.getPolicyDeltas() + "]"); } - servicePolicies.setPolicyVersion(servicePoliciesFromDb.getPolicyVersion()); final List<RangerPolicy> policies = servicePolicies.getPolicies() == null ? new ArrayList<>() : servicePolicies.getPolicies(); final List<RangerPolicy> newPolicies = RangerPolicyDeltaUtil.applyDeltas(policies, servicePoliciesFromDb.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); servicePolicies.setPolicies(newPolicies); + servicePolicies.setPolicyVersion(servicePoliciesFromDb.getPolicyVersion()); checkCacheSanity(serviceName, serviceStore, false); diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java index ff1165480..d84d772a9 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java +++ b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java @@ -186,13 +186,22 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron LOG.debug("Failed to execute runnable " + runnable + "because of OpmimisticLockException"); } } catch (Throwable e) { - LOG.error("Failed to execute runnable " + runnable, e); + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to execute runnable " + runnable, e); + } } return result; } }); isThisTransactionCommitted = result == runnable; + if (isParentTransactionCommitted) { + if (!isThisTransactionCommitted) { + LOG.info("Failed to commit runnable:[" + runnable + "]. Will retry!"); + } else { + LOG.info("Committed runnable:[" + runnable + "]."); + } + } } catch (OptimisticLockException optimisticLockException) { if (LOG.isDebugEnabled()) { @@ -203,7 +212,9 @@ public class RangerTransactionSynchronizationAdapter extends TransactionSynchron LOG.debug("Failed to commit TransactionService transaction, exception:[" + tse + "]"); } } catch (Throwable e){ - LOG.warn("Failed to commit TransactionService transaction, throwable:[" + e + "]"); + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to commit TransactionService transaction, throwable:[" + e + "]"); + } } } while (isParentTransactionCommitted && !isThisTransactionCommitted); }