This is an automated email from the ASF dual-hosted git repository. pradeep pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push: new 71744d3 RANGER-3540: Add support to read audit logs from Amazon CloudWatch 71744d3 is described below commit 71744d3ff8e0db7f0a49881a891e77edf0ea518d Author: pradeep <prad...@apache.org> AuthorDate: Tue Dec 21 17:32:17 2021 +0530 RANGER-3540: Add support to read audit logs from Amazon CloudWatch --- agents-audit/pom.xml | 2 +- .../AmazonCloudWatchAuditDestination.java | 38 ++- .../org/apache/ranger/audit/provider/MiscUtil.java | 72 ++++++ hbase-agent/conf/ranger-hbase-audit-changes.cfg | 1 + hbase-agent/scripts/install.properties | 1 + hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 1 + hdfs-agent/scripts/install.properties | 1 + hive-agent/conf/ranger-hive-audit-changes.cfg | 1 + hive-agent/scripts/install.properties | 1 + kms/scripts/install.properties | 1 + knox-agent/conf/ranger-knox-audit-changes.cfg | 1 + knox-agent/scripts/install.properties | 1 + plugin-atlas/conf/ranger-atlas-audit-changes.cfg | 1 + plugin-atlas/scripts/install.properties | 1 + .../conf/ranger-elasticsearch-audit-changes.cfg | 1 + plugin-elasticsearch/scripts/install.properties | 1 + plugin-kafka/conf/ranger-kafka-audit-changes.cfg | 1 + plugin-kafka/scripts/install.properties | 1 + plugin-kms/conf/ranger-kms-audit-changes.cfg | 1 + plugin-kylin/conf/ranger-kylin-audit-changes.cfg | 1 + plugin-kylin/scripts/install.properties | 1 + plugin-ozone/conf/ranger-ozone-audit-changes.cfg | 1 + plugin-ozone/scripts/install.properties | 1 + plugin-presto/conf/ranger-presto-audit-changes.cfg | 1 + plugin-presto/scripts/install.properties | 1 + plugin-solr/conf/ranger-solr-audit-changes.cfg | 1 + plugin-solr/scripts/install.properties | 1 + plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg | 1 + plugin-sqoop/scripts/install.properties | 1 + plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 1 + plugin-yarn/scripts/install.properties | 1 + pom.xml | 1 + security-admin/pom.xml | 15 ++ security-admin/scripts/install.properties | 7 +- .../scripts/ranger-admin-site-template.xml | 12 + security-admin/scripts/setup.sh | 29 +++ security-admin/scripts/upgrade_admin.py | 3 + .../org/apache/ranger/AccessAuditsService.java | 4 + .../cloudwatch/CloudWatchAccessAuditsService.java | 288 +++++++++++++++++++++ .../ranger/amazon/cloudwatch/CloudWatchMgr.java | 77 ++++++ .../ranger/amazon/cloudwatch/CloudWatchUtil.java | 258 ++++++++++++++++++ .../main/java/org/apache/ranger/biz/AssetMgr.java | 6 + .../java/org/apache/ranger/biz/RangerBizUtil.java | 1 + .../main/java/org/apache/ranger/biz/XAuditMgr.java | 8 + .../ElasticSearchAccessAuditsService.java | 17 +- .../ranger/elasticsearch/ElasticSearchUtil.java | 54 ---- .../ranger/solr/SolrAccessAuditsService.java | 17 +- .../main/java/org/apache/ranger/solr/SolrUtil.java | 52 ---- .../main/resources/conf.dist/ranger-admin-site.xml | 12 + storm-agent/conf/ranger-storm-audit-changes.cfg | 1 + storm-agent/scripts/install.properties | 1 + 51 files changed, 868 insertions(+), 135 deletions(-) diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml index 5d031cc..1519249 100644 --- a/agents-audit/pom.xml +++ b/agents-audit/pom.xml @@ -36,7 +36,7 @@ <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-bom</artifactId> - <version>1.11.327</version> + <version>${com.amazonaws.aws-java-sdk-bom.version}</version> <type>pom</type> <scope>import</scope> </dependency> diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java index b236a26..b3c7af4 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/AmazonCloudWatchAuditDestination.java @@ -31,6 +31,9 @@ import com.amazonaws.services.logs.model.InputLogEvent; import com.amazonaws.services.logs.model.InvalidSequenceTokenException; import com.amazonaws.services.logs.model.PutLogEventsRequest; import com.amazonaws.services.logs.model.PutLogEventsResult; +import com.amazonaws.services.logs.model.ResourceNotFoundException; + +import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.model.AuditEventBase; @@ -61,21 +64,22 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination { public static final String PROP_LOG_GROUP_NAME = "log_group"; public static final String PROP_LOG_STREAM_PREFIX = "log_stream_prefix"; public static final String CONFIG_PREFIX = "ranger.audit.amazon_cloudwatch"; + public static final String PROP_REGION = "region"; private String logGroupName; private String logStreamName; private AWSLogs logsClient; private String sequenceToken; + private String regionName; @Override public void init(Properties props, String propPrefix) { LOG.info("init() called for CloudWatchAuditDestination"); super.init(props, propPrefix); - this.logGroupName = MiscUtil.getStringProperty(props, propPrefix + "." - + PROP_LOG_GROUP_NAME); - this.logStreamName = MiscUtil.getStringProperty(props, propPrefix + "." - + PROP_LOG_STREAM_PREFIX) + MiscUtil.generateUniqueId(); + this.logGroupName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG_GROUP_NAME, "ranger_audits"); + this.logStreamName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_LOG_STREAM_PREFIX) + MiscUtil.generateUniqueId(); + this.regionName = MiscUtil.getStringProperty(props, propPrefix + "." + PROP_REGION); logsClient = getClient(); // Initialize client createLogStream(); @@ -95,8 +99,11 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination { PutLogEventsRequest req = new PutLogEventsRequest() .withLogEvents(toInputLogEvent(collection)) .withLogGroupName(logGroupName) - .withLogStreamName(logStreamName) - .withSequenceToken(sequenceToken); + .withLogStreamName(logStreamName); + + if (StringUtils.isNotBlank(sequenceToken)) { + req.setSequenceToken(sequenceToken); + } try { sequenceToken = pushLogEvents(req, false, client); @@ -117,6 +124,12 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination { try { PutLogEventsResult re = client.putLogEvents(req); sequenceToken = re.getNextSequenceToken(); + } catch (ResourceNotFoundException ex) { + if (!retryingOnInvalidSeqToken) { + createLogStream(); + return pushLogEvents(req, true, client); + } + throw ex; } catch (InvalidSequenceTokenException ex) { if (retryingOnInvalidSeqToken) { LOG.error("Unexpected invalid sequence token. Possible race condition occurred"); @@ -124,8 +137,9 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination { } // LogStream may exist before first push attempt, re-obtain the sequence token - LOG.info("Invalid sequence token. Plugin possibly restarted. " + - "Updating the sequence token and retrying"); + if (LOG.isDebugEnabled()) { + LOG.debug("Invalid sequence token. Plugin possibly restarted. Updating the sequence token and retrying"); + } sequenceToken = ex.getExpectedSequenceToken(); req.setSequenceToken(sequenceToken); return pushLogEvents(req, true, client); @@ -159,8 +173,7 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination { .withLogGroupName(logGroupName) .withLogStreamName(logStreamName); - LOG.info(String.format("Creating Log Stream `%s` in Log Group `%s`", - logStreamName, logGroupName)); + LOG.info(String.format("Creating Log Stream `%s` in Log Group `%s`", logStreamName, logGroupName)); client.createLogStream(req); } @@ -177,6 +190,9 @@ public class AmazonCloudWatchAuditDestination extends AuditDestination { } private AWSLogs newClient() { - return AWSLogsClientBuilder.standard().build(); + if (StringUtils.isBlank(regionName)) { + return AWSLogsClientBuilder.standard().build(); + } + return AWSLogsClientBuilder.standard().withRegion(regionName).build(); } } diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java index f58b813..268cb57 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MiscUtil.java @@ -24,6 +24,9 @@ import java.security.Principal; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.text.SimpleDateFormat; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Calendar; import java.util.Collections; @@ -885,4 +888,73 @@ public class MiscUtil { static final Random random = new Random(); } + // Utility methods + public static int toInt(Object value) { + if (value == null) { + return 0; + } + if (value instanceof Integer) { + return (Integer) value; + } + if (value.toString().isEmpty()) { + return 0; + } + try { + return Integer.valueOf(value.toString()); + } catch (Throwable t) { + logger.error("Error converting value to integer. Value = " + value, t); + } + return 0; + } + + public static long toLong(Object value) { + if (value == null) { + return 0; + } + if (value instanceof Long) { + return (Long) value; + } + if (value.toString().isEmpty()) { + return 0; + } + try { + return Long.valueOf(value.toString()); + } catch (Throwable t) { + logger.error("Error converting value to long. Value = " + value, t); + } + return 0; + } + + public static Date toDate(Object value) { + if (value == null) { + return null; + } + if (value instanceof Date) { + return (Date) value; + } + try { + // TODO: Do proper parsing based on Solr response value + return new Date(value.toString()); + } catch (Throwable t) { + logger.error("Error converting value to date. Value = " + value, t); + } + return null; + } + + public static Date toLocalDate(Object value) { + if (value == null) { + return null; + } + if (value instanceof Date) { + return (Date) value; + } + try { + LocalDateTime localDateTime = LocalDateTime.parse(value.toString(), DateTimeFormatter.ISO_DATE_TIME); + return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); + } catch (Throwable t) { + logger.error("Error converting value to date. Value = " + value, t); + } + return null; + } + } diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg index a6c7ffd..a743db4 100644 --- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg +++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg @@ -60,6 +60,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties index 87a2481..d105049 100644 --- a/hbase-agent/scripts/install.properties +++ b/hbase-agent/scripts/install.properties @@ -110,6 +110,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg index 92d2a4b..8e0e158 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg +++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg @@ -57,6 +57,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties index 323b878..698638b 100644 --- a/hdfs-agent/scripts/install.properties +++ b/hdfs-agent/scripts/install.properties @@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg index 52c715e..ec98baf 100644 --- a/hive-agent/conf/ranger-hive-audit-changes.cfg +++ b/hive-agent/conf/ranger-hive-audit-changes.cfg @@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties index 3720b66..e64e5e0 100644 --- a/hive-agent/scripts/install.properties +++ b/hive-agent/scripts/install.properties @@ -107,6 +107,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties index 6b6b662..4cf7908 100755 --- a/kms/scripts/install.properties +++ b/kms/scripts/install.properties @@ -227,6 +227,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg index 52c715e..ec98baf 100644 --- a/knox-agent/conf/ranger-knox-audit-changes.cfg +++ b/knox-agent/conf/ranger-knox-audit-changes.cfg @@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties index 4704004..cb1ccf8 100644 --- a/knox-agent/scripts/install.properties +++ b/knox-agent/scripts/install.properties @@ -102,6 +102,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg index 2d8251b..59c3927 100644 --- a/plugin-atlas/conf/ranger-atlas-audit-changes.cfg +++ b/plugin-atlas/conf/ranger-atlas-audit-changes.cfg @@ -38,6 +38,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists #log4j configuration xasecure.audit.log4j.is.enabled %XAAUDIT.LOG4J.ENABLE% mod create-if-not-exists diff --git a/plugin-atlas/scripts/install.properties b/plugin-atlas/scripts/install.properties index 3b777bd..c5b6eb8 100644 --- a/plugin-atlas/scripts/install.properties +++ b/plugin-atlas/scripts/install.properties @@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg index 52c715e..ec98baf 100644 --- a/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg +++ b/plugin-elasticsearch/conf/ranger-elasticsearch-audit-changes.cfg @@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-elasticsearch/scripts/install.properties b/plugin-elasticsearch/scripts/install.properties index 4111afe..fb2b40b 100644 --- a/plugin-elasticsearch/scripts/install.properties +++ b/plugin-elasticsearch/scripts/install.properties @@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg index bc5a089..94c2d24 100644 --- a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg +++ b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg @@ -51,6 +51,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-kafka/scripts/install.properties b/plugin-kafka/scripts/install.properties index 1e325e0..21f718b 100644 --- a/plugin-kafka/scripts/install.properties +++ b/plugin-kafka/scripts/install.properties @@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-kms/conf/ranger-kms-audit-changes.cfg b/plugin-kms/conf/ranger-kms-audit-changes.cfg index e5e9ae4..50e52a1 100644 --- a/plugin-kms/conf/ranger-kms-audit-changes.cfg +++ b/plugin-kms/conf/ranger-kms-audit-changes.cfg @@ -60,6 +60,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg index 52c715e..ec98baf 100644 --- a/plugin-kylin/conf/ranger-kylin-audit-changes.cfg +++ b/plugin-kylin/conf/ranger-kylin-audit-changes.cfg @@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-kylin/scripts/install.properties b/plugin-kylin/scripts/install.properties index 0134338..9d117ec 100644 --- a/plugin-kylin/scripts/install.properties +++ b/plugin-kylin/scripts/install.properties @@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg index 0eace6d..a3e2382 100644 --- a/plugin-ozone/conf/ranger-ozone-audit-changes.cfg +++ b/plugin-ozone/conf/ranger-ozone-audit-changes.cfg @@ -51,6 +51,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-ozone/scripts/install.properties b/plugin-ozone/scripts/install.properties index 1891d56..fcb74a9 100644 --- a/plugin-ozone/scripts/install.properties +++ b/plugin-ozone/scripts/install.properties @@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-presto/conf/ranger-presto-audit-changes.cfg b/plugin-presto/conf/ranger-presto-audit-changes.cfg index bc5a089..94c2d24 100644 --- a/plugin-presto/conf/ranger-presto-audit-changes.cfg +++ b/plugin-presto/conf/ranger-presto-audit-changes.cfg @@ -51,6 +51,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-presto/scripts/install.properties b/plugin-presto/scripts/install.properties index ce162a2..f8e8ac1 100644 --- a/plugin-presto/scripts/install.properties +++ b/plugin-presto/scripts/install.properties @@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-solr/conf/ranger-solr-audit-changes.cfg b/plugin-solr/conf/ranger-solr-audit-changes.cfg index ffa0a76..5ab43d6 100644 --- a/plugin-solr/conf/ranger-solr-audit-changes.cfg +++ b/plugin-solr/conf/ranger-solr-audit-changes.cfg @@ -52,6 +52,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-solr/scripts/install.properties b/plugin-solr/scripts/install.properties index d1852e6..1c292f1 100644 --- a/plugin-solr/scripts/install.properties +++ b/plugin-solr/scripts/install.properties @@ -105,6 +105,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg index 52c715e..ec98baf 100644 --- a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg +++ b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg @@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-sqoop/scripts/install.properties b/plugin-sqoop/scripts/install.properties index 81b4526..33d8813 100644 --- a/plugin-sqoop/scripts/install.properties +++ b/plugin-sqoop/scripts/install.properties @@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg index 52c715e..ec98baf 100644 --- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg +++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg @@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties index e73ab8b..f7eb036 100644 --- a/plugin-yarn/scripts/install.properties +++ b/plugin-yarn/scripts/install.properties @@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties diff --git a/pom.xml b/pom.xml index f9c46f6..5c621a5 100644 --- a/pom.xml +++ b/pom.xml @@ -230,6 +230,7 @@ <joda.time.version>2.10.6</joda.time.version> <jsonsmart.version>2.3.1</jsonsmart.version> <nimbus-jose-jwt.version>8.22.1</nimbus-jose-jwt.version> + <com.amazonaws.aws-java-sdk-bom.version>1.12.125</com.amazonaws.aws-java-sdk-bom.version> </properties> <profiles> <profile> diff --git a/security-admin/pom.xml b/security-admin/pom.xml index e9e9a53..bcce606 100644 --- a/security-admin/pom.xml +++ b/security-admin/pom.xml @@ -29,6 +29,17 @@ <properties> <skipJSTests>false</skipJSTests> </properties> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-bom</artifactId> + <version>${com.amazonaws.aws-java-sdk-bom.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> <dependencies> <dependency> <groupId>org.antlr</groupId> @@ -674,6 +685,10 @@ <artifactId>httpcore-nio</artifactId> <version>${httpcomponents.httpcore.version}</version> </dependency> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-logs</artifactId> + </dependency> </dependencies> <build> <pluginManagement> diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties index 5a8b00c..03f6cd7 100644 --- a/security-admin/scripts/install.properties +++ b/security-admin/scripts/install.properties @@ -83,7 +83,7 @@ rangerUsersync_password= keyadmin_password= -#Source for Audit Store. Currently solr and elasticsearch are supported. +#Source for Audit Store. Currently solr, elasticsearch and cloudwatch logs are supported. # * audit_store is solr audit_store=solr @@ -113,6 +113,11 @@ audit_solr_max_shards_per_node=1 audit_solr_acl_user_list_sasl=solr,infra-solr audit_solr_bootstrap_enabled=true +# * audit to amazon cloudwatch properties +audit_cloudwatch_region= +audit_cloudwatch_log_group= +audit_cloudwatch_log_stream_prefix= + #------------------------- DB CONFIG - END ---------------------------------- # diff --git a/security-admin/scripts/ranger-admin-site-template.xml b/security-admin/scripts/ranger-admin-site-template.xml index 72ff66e..037260f 100644 --- a/security-admin/scripts/ranger-admin-site-template.xml +++ b/security-admin/scripts/ranger-admin-site-template.xml @@ -177,6 +177,18 @@ <value></value> </property> <property> + <name>ranger.audit.amazon_cloudwatch.region</name> + <value></value> + </property> + <property> + <name>ranger.audit.amazon_cloudwatch.log_group</name> + <value></value> + </property> + <property> + <name>ranger.audit.amazon_cloudwatch.log_stream_prefix</name> + <value></value> + </property> + <property> <name>ranger.jpa.audit.jdbc.dialect</name> <value></value> </property> diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh index c3f51a0..9c5da5e 100755 --- a/security-admin/scripts/setup.sh +++ b/security-admin/scripts/setup.sh @@ -85,6 +85,9 @@ audit_solr_urls=$(get_prop 'audit_solr_urls' $PROPFILE) audit_solr_user=$(get_prop 'audit_solr_user' $PROPFILE) audit_solr_password=$(get_prop 'audit_solr_password' $PROPFILE) audit_solr_zookeepers=$(get_prop 'audit_solr_zookeepers' $PROPFILE) +audit_cloudwatch_region=$(get_prop 'audit_cloudwatch_region' $PROPFILE) +audit_cloudwatch_log_group=$(get_prop 'audit_cloudwatch_log_group' $PROPFILE) +audit_cloudwatch_log_stream_prefix=$(get_prop 'audit_cloudwatch_log_stream_prefix' $PROPFILE) policymgr_external_url=$(get_prop 'policymgr_external_url' $PROPFILE) policymgr_http_enabled=$(get_prop 'policymgr_http_enabled' $PROPFILE) policymgr_https_keystore_file=$(get_prop 'policymgr_https_keystore_file' $PROPFILE) @@ -269,6 +272,17 @@ init_variables(){ fi fi + if [ "${audit_store}" == "cloudwatch" ] ;then + if [ "${audit_cloudwatch_region}" == "" ] ;then + log "[I] Please provide valid region for 'amazon cloudwatch' audit store!" + exit 1 + fi + if [ "${audit_cloudwatch_log_group}" == "" ] ;then + log "[I] Please provide valid log-group for 'amazon cloudwatch' audit store!" + exit 1 + fi + fi + db_ssl_enabled=`echo $db_ssl_enabled | tr '[:upper:]' '[:lower:]'` if [ "${db_ssl_enabled}" != "true" ] then @@ -800,6 +814,21 @@ update_properties() { fi + if [ "${audit_store}" == "cloudwatch" ] + then + propertyName=ranger.audit.amazon_cloudwatch.region + newPropertyValue=${audit_cloudwatch_region} + updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger + + propertyName=ranger.audit.amazon_cloudwatch.log_group + newPropertyValue=${audit_cloudwatch_log_group} + updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger + + propertyName=ranger.audit.amazon_cloudwatch.log_stream_prefix + newPropertyValue=${audit_cloudwatch_log_stream_prefix} + updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger + fi + if [ "${audit_store}" != "" ] then propertyName=ranger.audit.source.type diff --git a/security-admin/scripts/upgrade_admin.py b/security-admin/scripts/upgrade_admin.py index 10fa485..85f57b8 100755 --- a/security-admin/scripts/upgrade_admin.py +++ b/security-admin/scripts/upgrade_admin.py @@ -116,6 +116,9 @@ config2xmlMAP = { 'audit_elasticsearch_port':'ranger.audit.elasticsearch.port', 'audit_elasticsearch_user':'ranger.audit.elasticsearch.user', 'audit_elasticsearch_password':'ranger.audit.elasticsearch.password', + 'audit_cloudwatch_region':'ranger.audit.amazon_cloudwatch.region', + 'audit_cloudwatch_log_group':'ranger.audit.amazon_cloudwatch.log_group', + 'audit_cloudwatch_log_stream_prefix':'ranger.audit.amazon_cloudwatch.log_stream_prefix', 'audit_solr_urls':'ranger.audit.solr.urls', 'auditDB.jdbc.dialect':'ranger.jpa.audit.jdbc.dialect', 'auditDB.jdbc.driver':'ranger.jpa.audit.jdbc.driver', diff --git a/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java index 4d97f28..de1feed 100644 --- a/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/AccessAuditsService.java @@ -77,6 +77,10 @@ public class AccessAuditsService { SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); searchFields.add(new SearchField("repoType", "repoType", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); + /* Note; search fields starting with '-' denotes exclude conditions, + * it should be handled manually if audit destination does not support the same. + * solr support this way while cloudwatch does not. + */ searchFields.add(new SearchField("-repoType", "-repoType", SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL)); searchFields.add(new SearchField("-requestUser", "-reqUser", diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java new file mode 100644 index 0000000..5c18016 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchAccessAuditsService.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.amazon.cloudwatch; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.common.JSONUtil; +import org.apache.ranger.common.MessageEnums; +import org.apache.ranger.common.PropertiesUtil; +import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.SearchCriteria; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.view.VXAccessAudit; +import org.apache.ranger.view.VXAccessAuditList; +import org.apache.ranger.view.VXLong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Service; + +import com.amazonaws.services.logs.AWSLogs; +import com.amazonaws.services.logs.model.FilteredLogEvent; + +@Service +@Scope("singleton") +public class CloudWatchAccessAuditsService extends org.apache.ranger.AccessAuditsService { + private static final Logger LOGGER = Logger.getLogger(CloudWatchAccessAuditsService.class); + + @Autowired + CloudWatchMgr cloudWatchMgr; + + @Autowired + CloudWatchUtil cloudWatchUtil; + + @Autowired + JSONUtil jsonUtil; + + public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) { + + final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true); + AWSLogs client = cloudWatchMgr.getClient(); + if (client == null) { + LOGGER.warn("CloudWatch client is null, so not running the query."); + throw restErrorUtil.createRESTException("Error connecting to cloudwatch", MessageEnums.ERROR_SYSTEM); + } + + List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>(); + Map<String, Object> paramList = searchCriteria.getParamList(); + updateUserExclusion(paramList); + + List<FilteredLogEvent> result; + try { + result = cloudWatchUtil.searchResources(client, searchCriteria, searchFields, sortFields); + } catch (Exception e) { + LOGGER.warn(String.format("CloudWatch query failed: %s", e.getMessage())); + throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM); + } + + VXAccessAuditList returnList = new VXAccessAuditList(); + if (result != null && CollectionUtils.isNotEmpty(result)) { + int recordCount = 0; + int endIndex = result.size() - 1; + endIndex = endIndex - searchCriteria.getStartIndex() < 0 ? endIndex : endIndex - searchCriteria.getStartIndex(); + for (int index = endIndex; recordCount < searchCriteria.getMaxRows() && index >=0 ; index--) { + FilteredLogEvent event = result.get(index); + AuthzAuditEvent auditEvent = null; + try { + auditEvent = MiscUtil.fromJson(event.getMessage(), AuthzAuditEvent.class); + } catch (Exception ex) { + LOGGER.error("Error while parsing json data" , ex); + } + VXAccessAudit vXAccessAudit = populateViewBean(auditEvent); + if (vXAccessAudit != null) { + String serviceType = vXAccessAudit.getServiceType(); + boolean isHive = "hive".equalsIgnoreCase(serviceType); + if (!hiveQueryVisibility && isHive) { + vXAccessAudit.setRequestData(null); + } else if (isHive) { + String accessType = vXAccessAudit.getAccessType(); + if ("grant".equalsIgnoreCase(accessType) || "revoke".equalsIgnoreCase(accessType)) { + String requestData = vXAccessAudit.getRequestData(); + if (requestData != null) { + try { + vXAccessAudit.setRequestData(java.net.URLDecoder.decode(requestData, "UTF-8")); + } catch (UnsupportedEncodingException e) { + LOGGER.warn("Error while encoding request data: " + requestData, e); + } + } else { + LOGGER.warn("Error in request data of audit from cloudwatch. AuditData: "+ vXAccessAudit.toString()); + } + } + } + } + xAccessAuditList.add(vXAccessAudit); + recordCount++; + } + returnList.setResultSize(result.size()); + returnList.setTotalCount(result.size()); + } + + returnList.setPageSize(searchCriteria.getMaxRows()); + returnList.setStartIndex(searchCriteria.getStartIndex()); + returnList.setVXAccessAudits(xAccessAuditList); + return returnList; + } + + public void setRestErrorUtil(RESTErrorUtil restErrorUtil) { + this.restErrorUtil = restErrorUtil; + } + + public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) { + long count = 100; + VXLong vXLong = new VXLong(); + vXLong.setValue(count); + return vXLong; + } + + private VXAccessAudit populateViewBean(AuthzAuditEvent auditEvent) { + VXAccessAudit accessAudit = new VXAccessAudit(); + + Object value = null; + if(LOGGER.isDebugEnabled()) { + LOGGER.debug("doc=" + auditEvent.toString()); + } + + value = auditEvent.getEventId(); + if (value != null) { + accessAudit.setId((long) value.hashCode()); + accessAudit.setEventId(value.toString()); + } + + value = auditEvent.getClusterName(); + if (value != null) { + accessAudit.setClusterName(value.toString()); + } + + value = auditEvent.getZoneName(); + if (value != null) { + accessAudit.setZoneName(value.toString()); + } + + value = auditEvent.getAgentHostname(); + if (value != null) { + accessAudit.setAgentHost(value.toString()); + } + + value = auditEvent.getPolicyVersion(); + if (value != null) { + accessAudit.setPolicyVersion(MiscUtil.toLong(value)); + } + + value = auditEvent.getAccessType(); + if (value != null) { + accessAudit.setAccessType(value.toString()); + } + + value = auditEvent.getAclEnforcer(); + if (value != null) { + accessAudit.setAclEnforcer(value.toString()); + } + + value = auditEvent.getAgentId(); + if (value != null) { + accessAudit.setAgentId(value.toString()); + } + + value = auditEvent.getRepositoryName(); + if (value != null) { + accessAudit.setRepoName(value.toString()); + XXService xxService = daoManager.getXXService().findByName(accessAudit.getRepoName()); + + if(xxService != null) { + accessAudit.setRepoDisplayName(xxService.getDisplayName()); + } + } + + value = auditEvent.getSessionId(); + if (value != null) { + accessAudit.setSessionId(value.toString()); + } + + value = auditEvent.getUser(); + if (value != null) { + accessAudit.setRequestUser(value.toString()); + } + + value = auditEvent.getRequestData(); + if (value != null) { + accessAudit.setRequestData(value.toString()); + } + value = auditEvent.getResourcePath(); + if (value != null) { + accessAudit.setResourcePath(value.toString()); + } + + value = auditEvent.getClientIP(); + if (value != null) { + accessAudit.setClientIP(value.toString()); + } + + value = auditEvent.getAccessResult(); + if (value != null) { + accessAudit.setAccessResult(MiscUtil.toInt(value)); + } + + value = auditEvent.getPolicyId(); + if (value != null) { + accessAudit.setPolicyId(MiscUtil.toLong(value)); + } + + value = auditEvent.getRepositoryType(); + if (value != null) { + accessAudit.setRepoType(MiscUtil.toInt(value)); + XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById((long) accessAudit.getRepoType()); + if (xServiceDef != null) { + accessAudit.setServiceType(xServiceDef.getName()); + accessAudit.setServiceTypeDisplayName(xServiceDef.getDisplayName()); + } + } + + value = auditEvent.getResourceType(); + if (value != null) { + accessAudit.setResourceType(value.toString()); + } + + value = auditEvent.getResultReason(); + if (value != null) { + accessAudit.setResultReason(value.toString()); + } + + value = auditEvent.getAction(); + if (value != null) { + accessAudit.setAction(value.toString()); + } + + value = auditEvent.getEventTime(); + if (value != null) { + accessAudit.setEventTime(MiscUtil.toLocalDate(value)); + } + + value = auditEvent.getSeqNum(); + if (value != null) { + accessAudit.setSequenceNumber(MiscUtil.toLong(value)); + } + + value = auditEvent.getEventCount(); + if (value != null) { + accessAudit.setEventCount(MiscUtil.toLong(value)); + } + + value = auditEvent.getEventDurationMS(); + if (value != null) { + accessAudit.setEventDuration(MiscUtil.toLong(value)); + } + + value = auditEvent.getTags(); + if (value != null) { + accessAudit.setTags(value.toString()); + } + + return accessAudit; + } + +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchMgr.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchMgr.java new file mode 100644 index 0000000..55d8229 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchMgr.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.amazon.cloudwatch; + +import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.CONFIG_PREFIX; +import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.PROP_REGION; + +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.common.PropertiesUtil; +import org.springframework.stereotype.Component; + +import com.amazonaws.services.logs.AWSLogs; +import com.amazonaws.services.logs.AWSLogsClientBuilder; + +/** + * This class initializes the CloudWatch client + * + */ +@Component +public class CloudWatchMgr { + + private static final Logger LOGGER = Logger.getLogger(CloudWatchMgr.class); + + private AWSLogs client = null; + private String regionName; + + synchronized void connect() { + if (client == null) { + synchronized (CloudWatchMgr.class) { + if (client == null) { + regionName = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + PROP_REGION); + try { + client = newClient(); + } catch (Throwable t) { + LOGGER.fatal("Can't connect to CloudWatch region: " + regionName, t); + } + } + } + } + } + + public AWSLogs getClient() { + if (client == null) { + synchronized (CloudWatchMgr.class) { + if (client == null) { + connect(); + } + } + } + return client; + } + + private AWSLogs newClient() { + if (StringUtils.isBlank(regionName)) { + return AWSLogsClientBuilder.standard().build(); + } + return AWSLogsClientBuilder.standard().withRegion(regionName).build(); + } +} diff --git a/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchUtil.java b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchUtil.java new file mode 100644 index 0000000..599a1ab --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/amazon/cloudwatch/CloudWatchUtil.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.amazon.cloudwatch; + +import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.CONFIG_PREFIX; +import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.PROP_LOG_GROUP_NAME; +import static org.apache.ranger.audit.destination.AmazonCloudWatchAuditDestination.PROP_LOG_STREAM_PREFIX; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.time.DateUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.common.PropertiesUtil; +import org.apache.ranger.common.SearchCriteria; +import org.apache.ranger.common.SearchField; +import org.apache.ranger.common.SearchField.SEARCH_TYPE; +import org.apache.ranger.common.SortField; +import org.apache.ranger.common.StringUtil; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.amazonaws.services.logs.AWSLogs; +import com.amazonaws.services.logs.model.FilterLogEventsRequest; +import com.amazonaws.services.logs.model.FilterLogEventsResult; +import com.amazonaws.services.logs.model.FilteredLogEvent; + +@Component +public class CloudWatchUtil { + private static final Logger LOGGER = Logger.getLogger(CloudWatchUtil.class); + + @Autowired + StringUtil stringUtil; + + String dateFormateStr = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + SimpleDateFormat dateFormat = new SimpleDateFormat(dateFormateStr); + private String logGroupName; + private String logStreamPrefix; + + public CloudWatchUtil() { + logGroupName = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + PROP_LOG_GROUP_NAME, "ranger_audits"); + logStreamPrefix = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + PROP_LOG_STREAM_PREFIX, ""); + String timeZone = PropertiesUtil.getProperty("ranger.cloudwatch.timezone"); + if (timeZone != null) { + LOGGER.info("Setting timezone to " + timeZone); + try { + dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone)); + } catch (Throwable t) { + LOGGER.error("Error setting timezone. TimeZone = " + timeZone); + } + } + } + + public List<FilteredLogEvent> searchResources(AWSLogs client, SearchCriteria searchCriteria, + List<SearchField> searchFields, List<SortField> sortFieldList) { + List<FilteredLogEvent> result = new ArrayList<FilteredLogEvent>(); + try { + String nextToken = null; + FilterLogEventsRequest filterLogEventsRequest = getFilterLogEventsRequest(client, searchCriteria, searchFields); + boolean done = false; + //TODO: Improve response time + //This approach is slow as cloudwatch doesn't provide timestamp based sorting in descending order + do { + if (nextToken != null) { + filterLogEventsRequest = filterLogEventsRequest.withNextToken(nextToken); + } + + FilterLogEventsResult response = client.filterLogEvents(filterLogEventsRequest); + if (response != null) { + if (CollectionUtils.isNotEmpty(response.getEvents())) { + //To handle outofmemory issue, max 10k records are stored in the list + if (result.size() > 10000) { + result.clear(); + } + result.addAll(response.getEvents()); + } else { + done = true; + break; + } + // check if token is the same + if (response.getNextToken().equals(nextToken)) { + done = true; + break; + } + // save new token + nextToken = response.getNextToken(); + if (nextToken == null) { + done = true; + break; + } + } + } while (!done); + LOGGER.info("Successfully got CloudWatch log events!"); + } catch (Exception e) { + LOGGER.error("Error searching records from CloudWatch", e); + } + return result; + } + + public FilterLogEventsRequest getFilterLogEventsRequest(AWSLogs client, SearchCriteria searchCriteria, + List<SearchField> searchFields) { + FilterLogEventsRequest filterLogEventsRequest = null; + StringBuilder filterPattern = new StringBuilder(""); + Date fromDate = null; + Date toDate = null; + + if (searchCriteria.getParamList() != null) { + List<String> filterExpr = new ArrayList<String>(); + + for (SearchField searchField : searchFields) { + Object paramValue = searchCriteria.getParamValue(searchField.getClientFieldName()); + if (paramValue == null || paramValue.toString().isEmpty()) { + continue; + } + + String fieldName = searchField.getFieldName(); + if (searchField.getDataType() == SearchField.DATA_TYPE.DATE) { + if (!(paramValue instanceof Date)) { + LOGGER.error("Search field is not a Java Date Object, paramValue = " + paramValue); + } else { + if (searchField.getSearchType() == SEARCH_TYPE.GREATER_EQUAL_THAN || searchField.getSearchType() == SEARCH_TYPE.GREATER_THAN) { + fromDate = (Date) paramValue; + } else if (searchField.getSearchType() == SEARCH_TYPE.LESS_EQUAL_THAN || searchField.getSearchType() == SEARCH_TYPE.LESS_THAN) { + toDate = (Date) paramValue; + } + } + } else if (paramValue instanceof Collection) { + String fq = orList(fieldName, (Collection<?>) paramValue); + if (StringUtils.isNotBlank(fq)) { + filterExpr.add(fq); + } + } else { + String fq = null; + if (searchField.getSearchType() == SEARCH_TYPE.PARTIAL) { + fq = setFieldForPartialSearch(fieldName, paramValue); + } else { + fq = setField(fieldName, paramValue); + } + if (StringUtils.isNotBlank(fq)) { + filterExpr.add(fq); + } + } + } + + if (fromDate == null) { + fromDate = DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH); + } + if (toDate == null) { + Date today = DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH); + toDate = DateUtils.addDays(today, 1); + } + + // Syntax : { ($.user.id = 1) && ($.users[0].email = "u...@example.com") } + if (CollectionUtils.isNotEmpty(filterExpr)) { + String strExpr = ""; + int count = -1; + for (String fq : filterExpr) { + count++; + if (count > 0) { + strExpr += " &&"; + } + strExpr = strExpr.concat("(" + fq + ")"); + } + if (strExpr.endsWith("&&")) { + strExpr = strExpr.substring(0, strExpr.length() - 3); + } + if (StringUtils.isNotBlank(strExpr)) { + filterPattern.append("{" + strExpr + "}"); + } + } + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("filterExpression for cloudwatch request " + filterPattern.toString()); + } + + // Add FilterPattern which will only fetch logs required + filterLogEventsRequest = new FilterLogEventsRequest() + .withLogGroupName(logGroupName) + .withStartTime(fromDate.getTime()) + .withEndTime(toDate.getTime()) + .withFilterPattern(filterPattern.toString()); + + if (StringUtils.isNotBlank(logStreamPrefix)) { + filterLogEventsRequest.setLogStreamNamePrefix(logStreamPrefix); + } + + return filterLogEventsRequest; + } + + //Syntax { $.user.email = "u...@example.com" || $.coordinates[0][1] = nonmatch && $.actions[2] = nomatch } + private String orList(String fieldName, Collection<?> valueList) { + if (valueList == null || valueList.isEmpty()) { + return null; + } + String expr = ""; + int count = -1; + for (Object value : valueList) { + count++; + if (count > 0) { + expr += " || "; + } + expr += setField(fieldName, value); + } + return expr; + } + + private String setField(String fieldName, Object value) { + if (value == null || StringUtils.isBlank(value.toString())) { + return null; + } + if (value instanceof Integer || value instanceof Long) { + if (fieldName.startsWith("-")) { + fieldName = fieldName.substring(1); + return "$." + fieldName + " != " + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()); + } + return "$." + fieldName + " = " + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()); + } + if (fieldName.startsWith("-")) { + fieldName = fieldName.substring(1); + return "$." + fieldName + " != \"" + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()) + "\""; + } + return "$." + fieldName + " = \"" + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()) + "\""; + } + + private String setFieldForPartialSearch(String fieldName, Object value) { + if (value == null || StringUtils.isBlank(value.toString())) { + return null; + } + return "$." + fieldName + "= \"*" + ClientUtils.escapeQueryChars(value.toString().trim().toLowerCase()) + "*\""; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java index d3ce251..36f137e 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java @@ -38,6 +38,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.apache.ranger.amazon.cloudwatch.CloudWatchAccessAuditsService; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.DateUtil; import org.apache.ranger.common.JSONUtil; @@ -114,6 +115,9 @@ public class AssetMgr extends AssetMgrBase { ElasticSearchAccessAuditsService elasticSearchAccessAuditsService; @Autowired + CloudWatchAccessAuditsService cloudWatchAccessAuditsService; + + @Autowired XPolicyService xPolicyService; @Autowired @@ -1128,6 +1132,8 @@ public class AssetMgr extends AssetMgrBase { return solrAccessAuditsService.searchXAccessAudits(searchCriteria); } else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(xaBizUtil.getAuditDBType())) { return elasticSearchAccessAuditsService.searchXAccessAudits(searchCriteria); + } else if (RangerBizUtil.AUDIT_STORE_CloudWatch.equalsIgnoreCase(xaBizUtil.getAuditDBType())) { + return cloudWatchAccessAuditsService.searchXAccessAudits(searchCriteria); } else { return xAccessAuditService.searchXAccessAudits(searchCriteria); } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index 75ebae6..2d6aa41 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -111,6 +111,7 @@ public class RangerBizUtil { public static final String AUDIT_STORE_RDBMS = "DB"; public static final String AUDIT_STORE_SOLR = "solr"; public static final String AUDIT_STORE_ElasticSearch = "elasticSearch"; + public static final String AUDIT_STORE_CloudWatch = "cloudwatch"; public static final boolean batchClearEnabled = PropertiesUtil.getBooleanProperty("ranger.jpa.jdbc.batch-clear.enable", true); public static final int policyBatchSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-clear.size", 10); public static final int batchPersistSize = PropertiesUtil.getIntProperty("ranger.jpa.jdbc.batch-persist.size", 500); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java index 4e5410e..9354350 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java @@ -21,6 +21,7 @@ package org.apache.ranger.biz; import javax.servlet.http.HttpServletResponse; +import org.apache.ranger.amazon.cloudwatch.CloudWatchAccessAuditsService; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.UserSessionBase; @@ -45,6 +46,9 @@ public class XAuditMgr extends XAuditMgrBase { ElasticSearchAccessAuditsService elasticSearchAccessAuditsService; @Autowired + CloudWatchAccessAuditsService cloudWatchAccessAuditsService; + + @Autowired RangerBizUtil rangerBizUtil; public VXTrxLog getXTrxLog(Long id) { @@ -119,6 +123,8 @@ public class XAuditMgr extends XAuditMgrBase { return solrAccessAuditsService.searchXAccessAudits(searchCriteria); } else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(auditDBType)) { return elasticSearchAccessAuditsService.searchXAccessAudits(searchCriteria); + } else if (RangerBizUtil.AUDIT_STORE_CloudWatch.equalsIgnoreCase(auditDBType)) { + return cloudWatchAccessAuditsService.searchXAccessAudits(searchCriteria); } else { return super.searchXAccessAudits(searchCriteria); } @@ -131,6 +137,8 @@ public class XAuditMgr extends XAuditMgrBase { return solrAccessAuditsService.getXAccessAuditSearchCount(searchCriteria); } else if (RangerBizUtil.AUDIT_STORE_ElasticSearch.equalsIgnoreCase(auditDBType)) { return elasticSearchAccessAuditsService.getXAccessAuditSearchCount(searchCriteria); + } else if (RangerBizUtil.AUDIT_STORE_CloudWatch.equalsIgnoreCase(auditDBType)) { + return cloudWatchAccessAuditsService.getXAccessAuditSearchCount(searchCriteria); } else { return super.getXAccessAuditSearchCount(searchCriteria); } diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java index 0b2e7df..5db858d 100644 --- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchAccessAuditsService.java @@ -20,6 +20,7 @@ package org.apache.ranger.elasticsearch; import org.apache.log4j.Logger; +import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; @@ -169,7 +170,7 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu value = source.get("policyVersion"); if (value != null) { - accessAudit.setPolicyVersion(elasticSearchUtil.toLong(value)); + accessAudit.setPolicyVersion(MiscUtil.toLong(value)); } value = source.get("access"); @@ -221,15 +222,15 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu //} value = source.get("result"); if (value != null) { - accessAudit.setAccessResult(elasticSearchUtil.toInt(value)); + accessAudit.setAccessResult(MiscUtil.toInt(value)); } value = source.get("policy"); if (value != null) { - accessAudit.setPolicyId(elasticSearchUtil.toLong(value)); + accessAudit.setPolicyId(MiscUtil.toLong(value)); } value = source.get("repoType"); if (value != null) { - accessAudit.setRepoType(elasticSearchUtil.toInt(value)); + accessAudit.setRepoType(MiscUtil.toInt(value)); if(null != daoManager) { XXServiceDefDao xxServiceDef = daoManager.getXXServiceDef(); if(xxServiceDef != null) { @@ -255,19 +256,19 @@ public class ElasticSearchAccessAuditsService extends org.apache.ranger.AccessAu } value = source.get("evtTime"); if (value != null) { - accessAudit.setEventTime(elasticSearchUtil.toDate(value)); + accessAudit.setEventTime(MiscUtil.toDate(value)); } value = source.get("seq_num"); if (value != null) { - accessAudit.setSequenceNumber(elasticSearchUtil.toLong(value)); + accessAudit.setSequenceNumber(MiscUtil.toLong(value)); } value = source.get("event_count"); if (value != null) { - accessAudit.setEventCount(elasticSearchUtil.toLong(value)); + accessAudit.setEventCount(MiscUtil.toLong(value)); } value = source.get("event_dur_ms"); if (value != null) { - accessAudit.setEventDuration(elasticSearchUtil.toLong(value)); + accessAudit.setEventDuration(MiscUtil.toLong(value)); } value = source.get("tags"); if (value != null) { diff --git a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java index 9bee640..777029e 100644 --- a/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/elasticsearch/ElasticSearchUtil.java @@ -68,60 +68,6 @@ public class ElasticSearchUtil { } } - - // Utility methods - public int toInt(Object value) { - if (value == null) { - return 0; - } - if (value instanceof Integer) { - return (Integer) value; - } - if (value.toString().isEmpty()) { - return 0; - } - try { - return Integer.valueOf(value.toString()); - } catch (Throwable t) { - logger.error("Error converting value to integer. Value = " + value, t); - } - return 0; - } - - public long toLong(Object value) { - if (value == null) { - return 0; - } - if (value instanceof Long) { - return (Long) value; - } - if (value.toString().isEmpty()) { - return 0; - } - try { - return Long.valueOf(value.toString()); - } catch (Throwable t) { - logger.error("Error converting value to long. Value = " + value, t); - } - return 0; - } - - public Date toDate(Object value) { - if (value == null) { - return null; - } - if (value instanceof Date) { - return (Date) value; - } - try { - LocalDateTime localDateTime = LocalDateTime.parse(value.toString(), DateTimeFormatter.ISO_DATE_TIME); - return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); - } catch (Throwable t) { - logger.error("Error converting value to date. Value = " + value, t); - } - return null; - } - public SearchResponse searchResources(SearchCriteria searchCriteria, List<SearchField> searchFields, List<SortField> sortFields, RestHighLevelClient client, String index) throws IOException { // See Also: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-query-builders.html QueryAccumulator queryAccumulator = new QueryAccumulator(searchCriteria); diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java index 0aea46d..2f6c869 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import org.apache.log4j.Logger; import org.apache.ranger.AccessAuditsService; +import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.common.MessageEnums; import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.RESTErrorUtil; @@ -153,7 +154,7 @@ public class SolrAccessAuditsService extends AccessAuditsService { value = doc.getFieldValue("policyVersion"); if (value != null) { - accessAudit.setPolicyVersion(solrUtil.toLong(value)); + accessAudit.setPolicyVersion(MiscUtil.toLong(value)); } value = doc.getFieldValue("access"); @@ -205,15 +206,15 @@ public class SolrAccessAuditsService extends AccessAuditsService { //} value = doc.getFieldValue("result"); if (value != null) { - accessAudit.setAccessResult(solrUtil.toInt(value)); + accessAudit.setAccessResult(MiscUtil.toInt(value)); } value = doc.getFieldValue("policy"); if (value != null) { - accessAudit.setPolicyId(solrUtil.toLong(value)); + accessAudit.setPolicyId(MiscUtil.toLong(value)); } value = doc.getFieldValue("repoType"); if (value != null) { - accessAudit.setRepoType(solrUtil.toInt(value)); + accessAudit.setRepoType(MiscUtil.toInt(value)); XXServiceDef xServiceDef = daoManager.getXXServiceDef().getById((long) accessAudit.getRepoType()); if (xServiceDef != null) { accessAudit.setServiceType(xServiceDef.getName()); @@ -234,19 +235,19 @@ public class SolrAccessAuditsService extends AccessAuditsService { } value = doc.getFieldValue("evtTime"); if (value != null) { - accessAudit.setEventTime(solrUtil.toDate(value)); + accessAudit.setEventTime(MiscUtil.toDate(value)); } value = doc.getFieldValue("seq_num"); if (value != null) { - accessAudit.setSequenceNumber(solrUtil.toLong(value)); + accessAudit.setSequenceNumber(MiscUtil.toLong(value)); } value = doc.getFieldValue("event_count"); if (value != null) { - accessAudit.setEventCount(solrUtil.toLong(value)); + accessAudit.setEventCount(MiscUtil.toLong(value)); } value = doc.getFieldValue("event_dur_ms"); if (value != null) { - accessAudit.setEventDuration(solrUtil.toLong(value)); + accessAudit.setEventDuration(MiscUtil.toLong(value)); } value = doc.getFieldValue("tags"); if (value != null) { diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java index 239698f..aa80e20 100644 --- a/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java @@ -302,56 +302,4 @@ public class SolrUtil { } } - // Utility methods - public int toInt(Object value) { - if (value == null) { - return 0; - } - if (value instanceof Integer) { - return (Integer) value; - } - if (value.toString().isEmpty()) { - return 0; - } - try { - return Integer.valueOf(value.toString()); - } catch (Throwable t) { - logger.error("Error converting value to integer. Value = " + value, t); - } - return 0; - } - - public long toLong(Object value) { - if (value == null) { - return 0; - } - if (value instanceof Long) { - return (Long) value; - } - if (value.toString().isEmpty()) { - return 0; - } - try { - return Long.valueOf(value.toString()); - } catch (Throwable t) { - logger.error("Error converting value to long. Value = " + value, t); - } - return 0; - } - - public Date toDate(Object value) { - if (value == null) { - return null; - } - if (value instanceof Date) { - return (Date) value; - } - try { - // TODO: Do proper parsing based on Solr response value - return new Date(value.toString()); - } catch (Throwable t) { - logger.error("Error converting value to date. Value = " + value, t); - } - return null; - } } diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml index d32a324..839cf18 100644 --- a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml +++ b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml @@ -75,6 +75,18 @@ <value>true</value> </property> <property> + <name>ranger.audit.amazon_cloudwatch.region</name> + <value>us-east-2</value> + </property> + <property> + <name>ranger.audit.amazon_cloudwatch.log_group</name> + <value>ranger_audits</value> + </property> + <property> + <name>ranger.audit.amazon_cloudwatch.log_stream_prefix</name> + <value></value> + </property> + <property> <name>ranger.audit.solr.urls</name> <value>http://##solr_host##:6083/solr/ranger_audits</value> <description></description> diff --git a/storm-agent/conf/ranger-storm-audit-changes.cfg b/storm-agent/conf/ranger-storm-audit-changes.cfg index 52c715e..ec98baf 100644 --- a/storm-agent/conf/ranger-storm-audit-changes.cfg +++ b/storm-agent/conf/ranger-storm-audit-changes.cfg @@ -58,6 +58,7 @@ xasecure.audit.destination.amazon_cloudwatch xasecure.audit.destination.amazon_cloudwatch.log_group %XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.log_stream_prefix %XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX% mod create-if-not-exists xasecure.audit.destination.amazon_cloudwatch.batch.filespool.dir %XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.amazon_cloudwatch.region %XAAUDIT.AMAZON_CLOUDWATCH.REGION% mod create-if-not-exists xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties index d219abf..58be387 100644 --- a/storm-agent/scripts/install.properties +++ b/storm-agent/scripts/install.properties @@ -104,6 +104,7 @@ XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE +XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE # End of V3 properties