AMBARI-18348. Ranger audit log collection is not used in Log Search (Miklos Gergely via oleewere)
Change-Id: I50b4c2c375d1657d620cfd1cab30943688ef079a Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d964480c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d964480c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d964480c Branch: refs/heads/branch-dev-logsearch Commit: d964480cc1efb804f71654559b6b70b115b82cf3 Parents: bc327fb Author: Miklos Gergely <mgerg...@hortonworks.com> Authored: Mon Oct 3 16:13:08 2016 +0200 Committer: oleewere <oleew...@gmail.com> Committed: Mon Oct 3 16:13:08 2016 +0200 ---------------------------------------------------------------------- .../ambari/logfeeder/filter/FilterKeyValue.java | 41 +++- .../logfeeder/filter/FilterKeyValueTest.java | 29 ++- .../configuration/logsearch-properties.xml | 19 +- .../LOGSEARCH/0.5.0/package/scripts/params.py | 5 + .../0.5.0/package/scripts/setup_logsearch.py | 19 +- .../templates/input.config-ambari.json.j2 | 235 ++++++++++++++++++- .../test/python/stacks/2.4/configs/default.json | 2 + ambari-web/app/data/HDP2/site_properties.js | 22 +- 8 files changed, 339 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java index c9c3f2c..5bb15ff 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java @@ -21,7 +21,7 @@ package org.apache.ambari.logfeeder.filter; import java.util.List; import java.util.Map; -import java.util.StringTokenizer; +import java.util.regex.Pattern; import org.apache.ambari.logfeeder.common.LogfeederException; import org.apache.ambari.logfeeder.input.InputMarker; @@ -37,7 +37,8 @@ public class FilterKeyValue extends Filter { private String sourceField = null; private String valueSplit = "="; private String fieldSplit = "\t"; - + private String valueBorders = null; + private MetricData errorMetric = new MetricData("filter.error.keyvalue", false); @Override @@ -47,6 +48,7 @@ public class FilterKeyValue extends Filter { sourceField = getStringValue("source_field"); valueSplit = getStringValue("value_split", valueSplit); fieldSplit = getStringValue("field_split", fieldSplit); + valueBorders = getStringValue("value_borders"); LOG.info("init() done. source_field=" + sourceField + ", value_split=" + valueSplit + ", " + ", field_split=" + fieldSplit + ", " + getShortDescription()); @@ -68,18 +70,16 @@ public class FilterKeyValue extends Filter { } Object valueObj = jsonObj.get(sourceField); if (valueObj != null) { - StringTokenizer fieldTokenizer = new StringTokenizer(valueObj.toString(), fieldSplit); - while (fieldTokenizer.hasMoreTokens()) { - String nv = fieldTokenizer.nextToken(); - StringTokenizer nvTokenizer = new StringTokenizer(nv, valueSplit); - while (nvTokenizer.hasMoreTokens()) { - String name = nvTokenizer.nextToken(); - if (nvTokenizer.hasMoreTokens()) { - String value = nvTokenizer.nextToken(); + String splitPattern = Pattern.quote(fieldSplit); + String[] tokens = valueObj.toString().split(splitPattern); + for (String nv : tokens) { + String[] nameValue = getNameValue(nv); + String name = nameValue != null && nameValue.length == 2 ? nameValue[0] : null; + String value = nameValue != null && nameValue.length == 2 ? nameValue[1] : null; + if (name != null && value != null) { jsonObj.put(name, value); - } else { - logParseError("name=" + name + ", pair=" + nv + ", field=" + sourceField + ", field_value=" + valueObj); - } + } else { + logParseError("name=" + name + ", pair=" + nv + ", field=" + sourceField + ", field_value=" + valueObj); } } } @@ -87,6 +87,21 @@ public class FilterKeyValue extends Filter { statMetric.value++; } + private String[] getNameValue(String nv) { + if (valueBorders != null) { + if (nv.charAt(nv.length() - 1) == valueBorders.charAt(1)) { + String splitPattern = Pattern.quote("" + valueBorders.charAt(0)); + return nv.substring(0, nv.length() - 1).split(splitPattern); + } else { + return null; + } + } + else { + String splitPattern = Pattern.quote(valueSplit); + return nv.split(splitPattern); + } + } + private void logParseError(String inputStr) { errorMetric.value++; String logMessageKey = this.getClass().getSimpleName() + "_PARSEERROR"; http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java index 30cee42..05647e6 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,8 +21,8 @@ package org.apache.ambari.logfeeder.filter; import java.util.HashMap; import java.util.Map; -import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.output.OutputManager; +import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.log4j.Logger; import org.easymock.Capture; import org.easymock.CaptureType; @@ -77,6 +77,31 @@ public class FilterKeyValueTest { } @Test + public void testFilterKeyValue_extractionWithBorders() throws Exception { + LOG.info("testFilterKeyValue_extractionWithBorders()"); + + Map<String, Object> config = new HashMap<String, Object>(); + config.put("source_field", "keyValueField"); + config.put("field_split", "&"); + config.put("value_borders", "()"); + init(config); + + mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); + EasyMock.expectLastCall(); + EasyMock.replay(mockOutputManager); + + filterKeyValue.apply("{ keyValueField: 'name1(value1)&name2(value2)' }", new InputMarker(null, null, 0)); + + EasyMock.verify(mockOutputManager); + Map<String, Object> jsonParams = capture.getValue(); + + assertEquals("Original missing!", "name1(value1)&name2(value2)", jsonParams.remove("keyValueField")); + assertEquals("Incorrect extraction: name1", "value1", jsonParams.remove("name1")); + assertEquals("Incorrect extraction: name2", "value2", jsonParams.remove("name2")); + assertTrue("jsonParams are not empty!", jsonParams.isEmpty()); + } + + @Test public void testFilterKeyValue_missingSourceField() throws Exception { LOG.info("testFilterKeyValue_missingSourceField()"); http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml index e521c8e..febeffd 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml @@ -113,6 +113,23 @@ <on-ambari-upgrade add="true"/> </property> <property> + <name>logsearch.solr.audit.logs.use.ranger</name> + <value>false</value> + <display-name>Ranger Audit Logs Enabled</display-name> + <description>Use Ranger Audit collection. This is supported only if Ranger Solr is installed in SolrCloud mode</description> + <value-attributes> + <type>boolean</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> + <property> + <name>logsearch.ranger.audit.logs.collection.name</name> + <value>ranger_audits</value> + <display-name>Log Search Solr Ranger Audit Logs Collection</display-name> + <description>Name for the ranger audit logs collection</description> + <on-ambari-upgrade add="true"/> + </property> + <property> <name>logsearch.logfeeder.include.default.level</name> <value>FATAL,ERROR,WARN</value> <description>Include default Log Feeder Log Levels for Log Search. Used for bootstrapping the configuration only. (levels: FATAL,ERROR,WARN,INFO,DEBUG,TRACE)</description> @@ -133,7 +150,7 @@ <on-ambari-upgrade add="true"/> </property> <property> - <name>logsearch.auth.external_auth.enable</name> + <name>logsearch.auth.external_auth.enabled</name> <value>true</value> <display-name>External authentication</display-name> <description>Enable external authentication</description> http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 8b0ce75..7200d7b 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -205,6 +205,11 @@ logsearch_properties['logsearch.solr.collection.history'] = 'history' logsearch_properties['logsearch.solr.history.config.name'] = 'history' logsearch_properties['logsearch.collection.history.replication.factor'] = '1' +if logsearch_properties['logsearch.solr.audit.logs.use.ranger'] == 'false': + del logsearch_properties['logsearch.ranger.audit.logs.collection.name'] + +del logsearch_properties['logsearch.solr.audit.logs.use.ranger'] + logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts']) logsearch_properties['logsearch.solr.jmx.port'] = infra_solr_jmx_port http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py index 8d84093..6c1a936 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py @@ -38,13 +38,14 @@ def setup_logsearch(): ) Directory([params.logsearch_dir, params.logsearch_server_conf, params.logsearch_config_set_dir], - mode=0755, - cd_access='a', - owner=params.logsearch_user, - group=params.user_group, - create_parents=True, - recursive_ownership=True - ) + mode=0755, + cd_access='a', + owner=params.logsearch_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True + ) + File(params.logsearch_log, mode=0644, owner=params.logsearch_user, @@ -53,8 +54,8 @@ def setup_logsearch(): ) PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"), - properties=params.logsearch_properties - ) + properties=params.logsearch_properties + ) File(format("{logsearch_server_conf}/HadoopServiceConfig.json"), content=Template("HadoopServiceConfig.json.j2"), http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 index 34f4b30..e2646ff 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 @@ -52,8 +52,12 @@ }, { "type":"ambari_audit", - "rowtype":"service", + "rowtype":"audit", "add_fields":{ + "logType":"AmbariAudit", + "enforcer":"ambari-acl", + "repoType":"1", + "repo":"ambari", "level":"INFO" }, "path":"{{ambari_server_log_dir}}/ambari-audit.log" @@ -237,10 +241,10 @@ }, "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", - "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime},%{SPACE}%{GREEDYDATA:log_message}", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime},%{SPACE}%{GREEDYDATA:log_message}", "post_map_values":{ - "logtime":{ + "evtTime":{ "map_date":{ "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX" } @@ -249,6 +253,229 @@ } + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "ambari_audit" + ] + + } + + }, + "source_field":"log_message", + "field_split":", ", + "value_borders":"()", + "post_map_values":{ + "User":{ + "map_fieldname":{ + "new_fieldname":"reqUser" + } + }, + "Hostname":{ + "map_fieldname":{ + "new_fieldname":"host" + } + }, + "RemoteIp":{ + "map_fieldname":{ + "new_fieldname":"cliIP" + } + }, + "RequestType":{ + "map_fieldname":{ + "new_fieldname":"cliType" + } + }, + "RequestId":{ + "map_fieldname":{ + "new_fieldname":"request_id" + } + }, + "TaskId":{ + "map_fieldname":{ + "new_fieldname":"task_id" + } + }, + "Operation":{ + "map_fieldname":{ + "new_fieldname":"action" + } + }, + "url":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + }, + "ResourcePath":{ + "map_fieldname":{ + "new_fieldname":"resource" + } + }, + "Status":[ + { + "map_fieldvalue":{ + "pre_value":"Success", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Successfully queued", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"QUEUED", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"PENDING", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"COMPLETED", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"IN_PROGRESS", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Failed", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"Failed to queue", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING_FAILED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"HOLDING_TIMEDOUT", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"FAILED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"TIMEDOUT", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"ABORTED", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"SKIPPED_FAILED", + "post_value":"0" + } + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + } + ], + "ResultStatus":[ + { + "map_fieldvalue":{ + "pre_value":"200 OK", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"201 Created", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"202 Accepted", + "post_value":"1" + } + }, + { + "map_fieldvalue":{ + "pre_value":"400 Bad Request", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"401 Unauthorized", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"403 Forbidden", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"404 Not Found", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"409 Resource Conflict", + "post_value":"0" + } + }, + { + "map_fieldvalue":{ + "pre_value":"500 Internal Server Error", + "post_value":"0" + } + }, + { + "map_fieldname":{ + "new_fieldname":"result" + } + } + ] + + } + } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index 8ac6fa7..1ebbda9 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -289,6 +289,8 @@ "logsearch.collection.service.logs.replication.factor": "1", "logsearch.collection.audit.logs.numshards": "10", "logsearch.collection.audit.logs.replication.factor": "1", + "logsearch.solr.audit.logs.use.ranger": "false", + "logsearch.ranger.audit.logs.collection.name": "ranger_audits", "logsearch.solr.metrics.collector.hosts" : "{metrics_collector_hosts}", "logsearch.auth.external_auth.enabled" : "false", "logsearch.auth.external_auth.host_url" : "{ambari_server_auth_host_url}", http://git-wip-us.apache.org/repos/asf/ambari/blob/d964480c/ambari-web/app/data/HDP2/site_properties.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js index 9311e75..fac40c4 100644 --- a/ambari-web/app/data/HDP2/site_properties.js +++ b/ambari-web/app/data/HDP2/site_properties.js @@ -2208,33 +2208,47 @@ var hdp2properties = [ "index": 5 }, { - "name": "logsearch.solr.metrics.collector.hosts", + "name": "logsearch.solr.audit.logs.use.ranger", "serviceName": "LOGSEARCH", "filename": "logsearch-properties.xml", "category": "Advanced logsearch-properties", "index": 6 }, { - "name": "logsearch.auth.external_auth.enable", + "name": "logsearch.ranger.audit.logs.collection.name", "serviceName": "LOGSEARCH", "filename": "logsearch-properties.xml", "category": "Advanced logsearch-properties", "index": 7 }, { - "name": "logsearch.auth.external_auth.host_url", + "name": "logsearch.solr.metrics.collector.hosts", "serviceName": "LOGSEARCH", "filename": "logsearch-properties.xml", "category": "Advanced logsearch-properties", "index": 8 }, { - "name": "logsearch.auth.external_auth.login_url", + "name": "logsearch.auth.external_auth.enable", "serviceName": "LOGSEARCH", "filename": "logsearch-properties.xml", "category": "Advanced logsearch-properties", "index": 9 }, + { + "name": "logsearch.auth.external_auth.host_url", + "serviceName": "LOGSEARCH", + "filename": "logsearch-properties.xml", + "category": "Advanced logsearch-properties", + "index": 10 + }, + { + "name": "logsearch.auth.external_auth.login_url", + "serviceName": "LOGSEARCH", + "filename": "logsearch-properties.xml", + "category": "Advanced logsearch-properties", + "index": 11 + }, /*infra-solr-client-log4j*/ { "name": "infra_solr_client_log_dir",
