This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push: new b876ae715 RANGER-4380: updated purge API to support purging of x_policy_export_audit records b876ae715 is described below commit b876ae715cd82abff9131ac7e60249760b579a18 Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Sat Aug 26 11:04:03 2023 -0700 RANGER-4380: updated purge API to support purging of x_policy_export_audit records --- .../ranger/plugin/util/RangerPurgeResult.java | 88 +++++++++++++++++ .../main/java/org/apache/ranger/RangerClient.java | 14 +++ .../python/apache_ranger/client/ranger_client.py | 7 ++ .../java/org/apache/ranger/TestRangerClient.java | 14 +++ .../java/org/apache/ranger/biz/ServiceDBStore.java | 110 ++++++++++++++++----- .../apache/ranger/db/XXPolicyExportAuditDao.java | 18 ++++ .../java/org/apache/ranger/rest/PublicAPIsv2.java | 9 +- .../java/org/apache/ranger/rest/ServiceREST.java | 39 +++++--- .../main/resources/META-INF/jpa_named_queries.xml | 5 + 9 files changed, 259 insertions(+), 45 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPurgeResult.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPurgeResult.java new file mode 100644 index 000000000..a57e053f4 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPurgeResult.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.util; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + + +@JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY ) +@JsonIgnoreProperties(ignoreUnknown=true) +public class RangerPurgeResult { + private String recordType; + private Long totalRecordCount; + private Long purgedRecordCount; + + public RangerPurgeResult() { } + + public RangerPurgeResult(String recordType, Long totalRecordCount, Long purgedRecordCount) { + this.recordType = recordType; + this.totalRecordCount = totalRecordCount; + this.purgedRecordCount = purgedRecordCount; + } + + public String getRecordType() { + return recordType; + } + + public void setRecordType(String recordType) { + this.recordType = recordType; + } + + public Long getTotalRecordCount() { + return totalRecordCount; + } + + public void setTotalRecordCount(Long totalRecordCount) { + this.totalRecordCount = totalRecordCount; + } + + public Long getPurgedRecordCount() { + return purgedRecordCount; + } + + public void setPurgedRecordCount(Long purgedRecordCount) { + this.purgedRecordCount = purgedRecordCount; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPurgeResult={"); + + sb.append("recordType={").append(recordType).append("} "); + sb.append("totalRecordCount={").append(totalRecordCount).append("} "); + sb.append("purgedRecordCount={").append(purgedRecordCount).append("} "); + + sb.append("}"); + + return sb; + } + +} diff --git a/intg/src/main/java/org/apache/ranger/RangerClient.java b/intg/src/main/java/org/apache/ranger/RangerClient.java index 838dfa52c..8a67d0be1 100644 --- a/intg/src/main/java/org/apache/ranger/RangerClient.java +++ b/intg/src/main/java/org/apache/ranger/RangerClient.java @@ -22,6 +22,7 @@ import com.sun.jersey.api.client.GenericType; import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; +import org.apache.ranger.plugin.util.RangerPurgeResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.sun.jersey.api.client.ClientResponse; @@ -49,6 +50,8 @@ public class RangerClient { private static final String PARAM_POLICY_NAME = "policyname"; private static final String PARAM_SERVICE_NAME = "serviceName"; private static final String PARAM_ZONE_NAME = "zoneName"; + private static final String PARAM_PURGE_RECORD_TYPE = "type"; + private static final String PARAM_PURGE_RETENTION_DAYS = "retentionDays"; private static final String PARAM_RELOAD_SERVICE_POLICIES_CACHE = "reloadServicePoliciesCache"; @@ -87,6 +90,7 @@ public class RangerClient { private static final String URI_SERVICE_TAGS = URI_SERVICE + "/%s/tags"; private static final String URI_PLUGIN_INFO = URI_BASE + "/plugins/info"; private static final String URI_POLICY_DELTAS = URI_BASE + "/server/policydeltas"; + private static final String URI_PURGE_RECORDS = URI_BASE + "/server/purge/records"; // APIs @@ -146,6 +150,7 @@ public class RangerClient { public static final API GET_SERVICE_TAGS = new API(URI_SERVICE_TAGS, HttpMethod.GET, Response.Status.OK); public static final API GET_PLUGIN_INFO = new API(URI_PLUGIN_INFO, HttpMethod.GET, Response.Status.OK); public static final API DELETE_POLICY_DELTAS = new API(URI_POLICY_DELTAS, HttpMethod.DELETE, Response.Status.NO_CONTENT); + public static final API PURGE_RECORDS = new API(URI_PURGE_RECORDS, HttpMethod.DELETE, Response.Status.OK); private final RangerRESTClient restClient; @@ -456,6 +461,15 @@ public class RangerClient { callAPI(DELETE_POLICY_DELTAS, queryParams); } + public List<RangerPurgeResult> purgeRecords(String recordType, int retentionDays) throws RangerServiceException { + Map<String, String> queryParams = new HashMap<>(); + + queryParams.put(PARAM_PURGE_RECORD_TYPE, recordType); + queryParams.put(PARAM_PURGE_RETENTION_DAYS, String.valueOf(retentionDays)); + + return callAPI(PURGE_RECORDS, queryParams, null, new GenericType<List<RangerPurgeResult>>(){}); + } + private ClientResponse invokeREST(API api, Map<String, String> params, Object request) throws RangerServiceException { final ClientResponse clientResponse; try { diff --git a/intg/src/main/python/apache_ranger/client/ranger_client.py b/intg/src/main/python/apache_ranger/client/ranger_client.py index 9731c266c..484a42128 100644 --- a/intg/src/main/python/apache_ranger/client/ranger_client.py +++ b/intg/src/main/python/apache_ranger/client/ranger_client.py @@ -299,6 +299,9 @@ class RangerClient: def delete_policy_deltas(self, days, reloadServicePoliciesCache): self.client_http.call_api(RangerClient.DELETE_POLICY_DELTAS, { 'days': days, 'reloadServicePoliciesCache': reloadServicePoliciesCache}) + def purge_records(self, record_type, retention_days): + return self.client_http.call_api(RangerClient.PURGE_RECORDS, { 'type': record_type, 'retentionDays': retention_days}) + @@ -338,6 +341,7 @@ class RangerClient: URI_SERVICE_TAGS = URI_SERVICE + "/{serviceName}/tags" URI_PLUGIN_INFO = URI_BASE + "/plugins/info" URI_POLICY_DELTAS = URI_BASE + "/server/policydeltas" + URI_PURGE_RECORDS = URI_BASE + "/server/purge/records" # APIs CREATE_SERVICEDEF = API(URI_SERVICEDEF, HttpMethod.POST, HTTPStatus.OK) @@ -397,6 +401,7 @@ class RangerClient: GET_SERVICE_TAGS = API(URI_SERVICE_TAGS, HttpMethod.GET, HTTPStatus.OK) GET_PLUGIN_INFO = API(URI_PLUGIN_INFO, HttpMethod.GET, HTTPStatus.OK) DELETE_POLICY_DELTAS = API(URI_POLICY_DELTAS, HttpMethod.DELETE, HTTPStatus.NO_CONTENT) + PURGE_RECORDS = API(URI_PURGE_RECORDS, HttpMethod.DELETE, HTTPStatus.OK) @@ -531,6 +536,8 @@ class RangerClientHttp: elif response.status_code == HTTPStatus.NOT_FOUND: LOG.error("Not found. HTTP Status: %s", HTTPStatus.NOT_FOUND) + ret = None + elif response.status_code == HTTPStatus.NOT_MODIFIED: ret = None else: raise RangerServiceException(api, response) diff --git a/intg/src/test/java/org/apache/ranger/TestRangerClient.java b/intg/src/test/java/org/apache/ranger/TestRangerClient.java index 56e6fbbeb..7da6b18a5 100644 --- a/intg/src/test/java/org/apache/ranger/TestRangerClient.java +++ b/intg/src/test/java/org/apache/ranger/TestRangerClient.java @@ -23,6 +23,7 @@ import org.apache.ranger.plugin.model.RangerSecurityZone; import org.apache.ranger.plugin.model.RangerSecurityZoneHeaderInfo; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceHeaderInfo; +import org.apache.ranger.plugin.util.RangerPurgeResult; import org.apache.ranger.plugin.util.RangerRESTClient; import org.junit.Assert; import org.junit.Test; @@ -202,4 +203,17 @@ public class TestRangerClient { Assert.assertEquals(Collections.emptyList(), securityZones); } + + @Test + public void testPurgeRecords() throws RangerServiceException { + RangerClient client = Mockito.mock(RangerClient.class); + String recordType = "login_records"; + int retentionDays = 180; + + when(client.purgeRecords(recordType, retentionDays)).thenReturn(Collections.emptyList()); + + List<RangerPurgeResult> purgeResults = client.purgeRecords(recordType, retentionDays); + + Assert.assertEquals(Collections.emptyList(), purgeResults); + } } \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 9b02229e1..8f1174ac4 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -67,11 +67,14 @@ import org.apache.ranger.common.GUIDUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.RangerCommonEnums; import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter; +import org.apache.ranger.db.XXAuthSessionDao; import org.apache.ranger.db.XXGlobalStateDao; import org.apache.ranger.db.XXPolicyDao; +import org.apache.ranger.db.XXTrxLogDao; import org.apache.ranger.entity.XXTagChangeLog; import org.apache.ranger.plugin.model.RangerSecurityZone; import org.apache.ranger.plugin.util.RangerCommonConstants; +import org.apache.ranger.plugin.util.RangerPurgeResult; import org.apache.ranger.plugin.util.ServiceTags; import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator; import org.apache.ranger.plugin.model.validation.RangerValidator; @@ -102,6 +105,7 @@ import org.apache.ranger.db.XXDataMaskTypeDefDao; import org.apache.ranger.db.XXEnumDefDao; import org.apache.ranger.db.XXEnumElementDefDao; import org.apache.ranger.db.XXPolicyConditionDefDao; +import org.apache.ranger.db.XXPolicyExportAuditDao; import org.apache.ranger.db.XXPolicyLabelMapDao; import org.apache.ranger.db.XXResourceDefDao; import org.apache.ranger.db.XXServiceConfigDefDao; @@ -255,6 +259,8 @@ public class ServiceDBStore extends AbstractServiceStore { public static Integer LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = 0; public static boolean SUPPORTS_PURGE_TRANSACTION_RECORDS = false; public static Integer TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS = 0; + public static boolean SUPPORTS_PURGE_POLICY_EXPORT_LOGS = false; + public static Integer POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS = 0; private static final String RANGER_PLUGIN_CONFIG_PREFIX = "ranger.plugin."; public static final String RANGER_PLUGIN_AUDIT_FILTERS = "ranger.plugin.audit.filters"; @@ -398,10 +404,12 @@ public class ServiceDBStore extends AbstractServiceStore { RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.delta.retention.time.in.days", 7); TAG_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.tag.delta.retention.time.in.days", 3); - SUPPORTS_PURGE_LOGIN_RECORDS = config.getBoolean("ranger.admin.init.purge.login_records", false); - LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.login_records.retention.days", 0); + SUPPORTS_PURGE_LOGIN_RECORDS = config.getBoolean("ranger.admin.init.purge.login_records", false); SUPPORTS_PURGE_TRANSACTION_RECORDS = config.getBoolean("ranger.admin.init.purge.transaction_records", false); + SUPPORTS_PURGE_POLICY_EXPORT_LOGS = config.getBoolean("ranger.admin.init.purge.policy_export_logs", false); + LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.login_records.retention.days", 0); TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.transaction_records.retention.days", 0); + POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.policy_export_logs.retention.days", 0); isRolesDownloadedByService = config.getBoolean("ranger.support.for.service.specific.role.download", false); SUPPORTS_IN_PLACE_POLICY_UPDATES = SUPPORTS_POLICY_DELTAS && config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES, RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT); @@ -413,6 +421,8 @@ public class ServiceDBStore extends AbstractServiceStore { LOG.info("LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS=" + LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS); LOG.info("SUPPORTS_PURGE_TRANSACTION_RECORDS=" + SUPPORTS_PURGE_TRANSACTION_RECORDS); LOG.info("TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS=" + TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS); + LOG.info("SUPPORTS_PURGE_POLICY_EXPORT_LOGS=" + SUPPORTS_PURGE_POLICY_EXPORT_LOGS); + LOG.info("POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS=" + POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS); LOG.info("isRolesDownloadedByService=" + isRolesDownloadedByService); LOG.info("SUPPORTS_IN_PLACE_POLICY_UPDATES=" + SUPPORTS_IN_PLACE_POLICY_UPDATES); @@ -430,12 +440,21 @@ public class ServiceDBStore extends AbstractServiceStore { createGenericUsers(); resetPolicyUpdateLog(RETENTION_PERIOD_IN_DAYS, RangerPolicyDelta.CHANGE_TYPE_RANGER_ADMIN_START); resetTagUpdateLog(TAG_RETENTION_PERIOD_IN_DAYS, ServiceTags.TagsChangeType.RANGER_ADMIN_START); + + List<RangerPurgeResult> purgeResults = new ArrayList<>(); + if (SUPPORTS_PURGE_LOGIN_RECORDS) { - removeAuthSessions(LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS); + removeAuthSessions(LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS, purgeResults); } + if (SUPPORTS_PURGE_TRANSACTION_RECORDS) { - removeTransactionLogs(TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS); + removeTransactionLogs(TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS, purgeResults); + } + + if (SUPPORTS_PURGE_POLICY_EXPORT_LOGS) { + removePolicyExportLogs(POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS, purgeResults); } + //createUnzonedSecurityZone(); initRMSDaos(); return null; @@ -5326,53 +5345,90 @@ public class ServiceDBStore extends AbstractServiceStore { } } - public void removeAuthSessions(int retentionInDays) { + public void removeAuthSessions(int retentionInDays, List<RangerPurgeResult> result) { if (LOG.isDebugEnabled()) { LOG.debug("==> removeAuthSessions(" + retentionInDays + ")"); } if (retentionInDays > 0) { - long rowsCount = daoMgr.getXXAuthSession().getAllCount(); - long rowsDeleted = daoMgr.getXXAuthSession().deleteOlderThan(retentionInDays); + XXAuthSessionDao dao = daoMgr.getXXAuthSession(); + long rowsCount = dao.getAllCount(); + long rowsDeleted = dao.deleteOlderThan(retentionInDays); + LOG.info("Deleted " + rowsDeleted + " records from x_auth_sess that are older than " + retentionInDays + " days"); - List<XXTrxLog> trxLogList = new ArrayList<XXTrxLog>(); - XXTrxLog xxTrxLog = new XXTrxLog(); - xxTrxLog.setAction("Deleted Auth Session records"); - xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_AUTH_SESS); - xxTrxLog.setPreviousValue("Total Records : "+rowsCount); - xxTrxLog.setNewValue("Deleted Records : "+rowsDeleted); - trxLogList.add(xxTrxLog); - bizUtil.createTrxLog(trxLogList); + + XXTrxLog trxLog = new XXTrxLog(); + + trxLog.setAction("Deleted Auth Session records"); + trxLog.setObjectClassType(AppConstants.CLASS_TYPE_AUTH_SESS); + trxLog.setPreviousValue("Total Records : " + rowsCount); + trxLog.setNewValue("Deleted Records : " + rowsDeleted); + + bizUtil.createTrxLog(Collections.singletonList(trxLog)); + + result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_LOGIN_LOGS, rowsCount, rowsDeleted)); } if (LOG.isDebugEnabled()) { LOG.debug("<== removeAuthSessions(" + retentionInDays + ")"); - } } - public void removeTransactionLogs(int retentionInDays) { + public void removeTransactionLogs(int retentionInDays, List<RangerPurgeResult> result) { if (LOG.isDebugEnabled()) { LOG.debug("==> removeTransactionLogs(" + retentionInDays + ")"); } if (retentionInDays > 0) { - long rowsCount = daoMgr.getXXTrxLog().getAllCount(); - long rowsDeleted = daoMgr.getXXTrxLog().deleteOlderThan(retentionInDays); + XXTrxLogDao dao = daoMgr.getXXTrxLog(); + long rowsCount = dao.getAllCount(); + long rowsDeleted = dao.deleteOlderThan(retentionInDays); + LOG.info("Deleted " + rowsDeleted + " records from x_trx_log that are older than " + retentionInDays + " days"); - List<XXTrxLog> trxLogList = new ArrayList<XXTrxLog>(); - XXTrxLog xxTrxLog = new XXTrxLog(); - xxTrxLog.setAction("Deleted Transaction records"); - xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_AUTH_SESS); - xxTrxLog.setPreviousValue("Total Records : "+rowsCount); - xxTrxLog.setNewValue("Deleted Records : "+rowsDeleted); - trxLogList.add(xxTrxLog); - bizUtil.createTrxLog(trxLogList); + + XXTrxLog trxLog = new XXTrxLog(); + + trxLog.setAction("Deleted Transaction records"); + trxLog.setObjectClassType(AppConstants.CLASS_TYPE_TRX_LOG); + trxLog.setPreviousValue("Total Records : " + rowsCount); + trxLog.setNewValue("Deleted Records : " + rowsDeleted); + + bizUtil.createTrxLog(Collections.singletonList(trxLog)); + + result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_TRX_LOGS, rowsCount, rowsDeleted)); } if (LOG.isDebugEnabled()) { LOG.debug("<== removeTransactionLogs(" + retentionInDays + ")"); + } + } + + public void removePolicyExportLogs(int retentionInDays, List<RangerPurgeResult> result) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> removePolicyExportLogs({})", retentionInDays); + } + + if (retentionInDays > 0) { + XXPolicyExportAuditDao dao = daoMgr.getXXPolicyExportAudit(); + long rowsCount = dao.getAllCount(); + long rowsDeleted = dao.deleteOlderThan(retentionInDays); + LOG.info("Deleted {} records from x_policy_export_audit that are older than {} days", rowsDeleted, retentionInDays); + + XXTrxLog trxLog = new XXTrxLog(); + + trxLog.setAction("Deleted policy export audit records"); + trxLog.setObjectClassType(AppConstants.CLASS_TYPE_XA_POLICY_EXPORT_AUDIT); + trxLog.setPreviousValue("Total Records: " + rowsCount); + trxLog.setNewValue("Deleted Records: " + rowsDeleted); + + bizUtil.createTrxLog(Collections.singletonList(trxLog)); + + result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_POLICY_EXPORT_LOGS, rowsCount, rowsDeleted)); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== removePolicyExportLogs({})", retentionInDays); } } diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyExportAuditDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyExportAuditDao.java index deed28e47..1d13feb02 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/XXPolicyExportAuditDao.java +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPolicyExportAuditDao.java @@ -21,13 +21,31 @@ import org.apache.ranger.common.db.BaseDao; import org.apache.ranger.entity.XXPolicyExportAudit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import java.util.Date; +import java.util.concurrent.TimeUnit; + @Service public class XXPolicyExportAuditDao extends BaseDao<XXPolicyExportAudit> { + private static final Logger logger = LoggerFactory.getLogger(XXPolicyExportAuditDao.class); public XXPolicyExportAuditDao( RangerDaoManagerBase daoManager ) { super(daoManager); } + + public long deleteOlderThan(int olderThanInDays) { + Date since = new Date(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(olderThanInDays)); + + logger.info("Deleting x_policy_export_audit records that are older than {} days, that is, older than {}", olderThanInDays, since); + + long ret = getEntityManager().createNamedQuery("XXPolicyExportAudit.deleteOlderThan").setParameter("olderThan", since).executeUpdate(); + + logger.info("Deleted x_policy_export_audit {} records", ret); + + return ret; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index 1bdac859c..cd906ed22 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -35,6 +35,7 @@ import org.apache.ranger.plugin.model.RangerServiceHeaderInfo; import org.apache.ranger.plugin.model.RangerServiceResource; import org.apache.ranger.plugin.model.RangerServiceTags; import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; +import org.apache.ranger.plugin.util.RangerPurgeResult; import org.apache.ranger.plugin.util.ServiceTags; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -900,15 +901,17 @@ public class PublicAPIsv2 { @DELETE @Path("/api/server/purge/records") @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") - public void purgeRecords(@QueryParam("type") String recordType, @DefaultValue("180") @QueryParam("retentionDays") Integer olderThan, @Context HttpServletRequest request) { + public List<RangerPurgeResult> purgeRecords(@QueryParam("type") String recordType, @DefaultValue("180") @QueryParam("retentionDays") Integer olderThan, @Context HttpServletRequest request) { if (logger.isDebugEnabled()) { logger.debug("==> PublicAPIsv2.purgeRecords(" + recordType + ", " + olderThan + ")"); } - serviceREST.purgeRecords(recordType, olderThan, request); + List<RangerPurgeResult> ret = serviceREST.purgeRecords(recordType, olderThan, request); if (logger.isDebugEnabled()) { - logger.debug("<== PublicAPIsv2.purgeRecords(" + recordType + ", " + olderThan + ")"); + logger.debug("<== PublicAPIsv2.purgeRecords(" + recordType + ", " + olderThan + "): ret=" + ret); } + + return ret; } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index 852c163df..429b9e5bc 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -124,6 +124,7 @@ import org.apache.ranger.plugin.util.GrantRevokeRequest; import org.apache.ranger.plugin.util.JsonUtilsV2; import org.apache.ranger.plugin.util.RangerAccessRequestUtil; import org.apache.ranger.plugin.util.RangerPerfTracer; +import org.apache.ranger.plugin.util.RangerPurgeResult; import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; import org.apache.ranger.security.context.RangerAPIList; @@ -188,6 +189,10 @@ public class ServiceREST { final static public String POLICY_MATCHING_ALGO_BY_RESOURCE = "matchByPolicySignature"; final static public String PARAM_POLICY_MATCHING_ALGORITHM = "policyMatchingAlgorithm"; + public static final String PURGE_RECORD_TYPE_LOGIN_LOGS = "login_records"; + public static final String PURGE_RECORD_TYPE_TRX_LOGS = "trx_records"; + public static final String PURGE_RECORD_TYPE_POLICY_EXPORT_LOGS = "policy_export_logs"; + @Autowired RESTErrorUtil restErrorUtil; @@ -3942,32 +3947,34 @@ public class ServiceREST { @DELETE @Path("/server/purge/records") @PreAuthorize("hasRole('ROLE_SYS_ADMIN')") - public void purgeRecords(@QueryParam("type") String recordType, @DefaultValue("180") @QueryParam("retentionDays") Integer olderThan, @Context HttpServletRequest request) { + public List<RangerPurgeResult> purgeRecords(@QueryParam("type") String recordType, @DefaultValue("180") @QueryParam("retentionDays") Integer olderThan, @Context HttpServletRequest request) { if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.purgeRecords(" + recordType + ", " + olderThan + ")"); } - if (!"login_records".equalsIgnoreCase(recordType) && !"trx_records".equalsIgnoreCase(recordType)) { - throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, "Invalid record type - " + recordType, true); - } - - if (olderThan < 1) { - throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, "Retention days can't be lesser than 1", true); - } - - RangerPerfTracer perf = null; + List<RangerPurgeResult> ret = new ArrayList<>(); + RangerPerfTracer perf = null; try { if (RangerPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = RangerPerfTracer.getPerfTracer(PERF_LOG, "ServiceREST.purgeRecords(recordType=" + recordType + ", olderThan=" + olderThan + ")"); } - if ("login_records".equalsIgnoreCase(recordType)) { - svcStore.removeAuthSessions(olderThan); - } else if ("trx_records".equalsIgnoreCase(recordType)) { - svcStore.removeTransactionLogs(olderThan); + if (olderThan < 1) { + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, "Retention days can't be lesser than 1", true); } + if (PURGE_RECORD_TYPE_LOGIN_LOGS.equalsIgnoreCase(recordType)) { + svcStore.removeAuthSessions(olderThan, ret); + } else if (PURGE_RECORD_TYPE_TRX_LOGS.equalsIgnoreCase(recordType)) { + svcStore.removeTransactionLogs(olderThan, ret); + } else if (PURGE_RECORD_TYPE_POLICY_EXPORT_LOGS.equalsIgnoreCase(recordType)) { + svcStore.removePolicyExportLogs(olderThan, ret); + } else { + throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, + recordType + ": invalid record type. Valid values: [ " + PURGE_RECORD_TYPE_LOGIN_LOGS + ", " + PURGE_RECORD_TYPE_TRX_LOGS + ", " + PURGE_RECORD_TYPE_POLICY_EXPORT_LOGS + " ]", + true); + } } catch (WebApplicationException excp) { throw excp; } catch (Throwable excp) { @@ -3978,8 +3985,10 @@ public class ServiceREST { } if (LOG.isDebugEnabled()) { - LOG.debug("<== ServiceREST.purgeRecords(" + recordType + ", " + olderThan + ")"); + LOG.debug("<== ServiceREST.purgeRecords(" + recordType + ", " + olderThan + "): ret=" + ret); } + + return ret; } private HashMap<String, Object> getCSRFPropertiesMap(HttpServletRequest request) { diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml index cf6ebad95..3718b051a 100755 --- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml +++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml @@ -453,6 +453,11 @@ <query>select obj from XXPolicy obj where obj.service = :serviceId and obj.resourceSignature = :resSignature</query> </named-query> + <!-- XXPolicyExportAudit --> + <named-query name="XXPolicyExportAudit.deleteOlderThan"> + <query>DELETE FROM XXPolicyExportAudit obj WHERE obj.createTime < :olderThan</query> + </named-query> + <!-- XXServiceDef --> <named-query name="XXServiceDef.findByName"> <query>select obj from XXServiceDef obj where obj.name = :name</query>