oleewere closed pull request #38: AMBARI-24943. Log Feeder: log level filter
does not have any affect if level field is null
URL: https://github.com/apache/ambari-logsearch/pull/38
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
index a1780cfe2c..a00e99b2ff 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java
@@ -30,7 +30,7 @@
import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigZKHelper;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheListener;
@@ -133,7 +133,7 @@ public boolean isAllowed(String hostName, String logId,
String level, List<Strin
}
public boolean isAllowed(String jsonBlock, InputMarker inputMarker,
List<String> defaultLogLevels) {
- if (org.apache.commons.lang3.StringUtils.isEmpty(jsonBlock)) {
+ if (StringUtils.isEmpty(jsonBlock)) {
return DEFAULT_VALUE;
}
Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
@@ -160,8 +160,8 @@ public boolean applyFilter(Map<String, Object> jsonObj,
List<String> defaultLogL
String hostName = (String) jsonObj.get(LogFeederConstants.SOLR_HOST);
String logId = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT);
- String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL);
- if (org.apache.commons.lang3.StringUtils.isNotBlank(hostName) &&
org.apache.commons.lang3.StringUtils.isNotBlank(logId) &&
org.apache.commons.lang3.StringUtils.isNotBlank(level)) {
+ String level = (String)
jsonObj.getOrDefault(LogFeederConstants.SOLR_LEVEL,
LogFeederConstants.LOG_LEVEL_UNKNOWN);
+ if (StringUtils.isNotBlank(hostName) && StringUtils.isNotBlank(logId) &&
StringUtils.isNotBlank(level)) {
return isAllowed(hostName, logId, level, defaultLogLevels);
} else {
return DEFAULT_VALUE;
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
index ff0805dad9..0b1c5a25fa 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.output;
import com.google.common.hash.Hashing;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.plugin.common.MetricData;
import org.apache.ambari.logfeeder.plugin.input.Input;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
@@ -42,7 +43,7 @@
private static final int MAX_OUTPUT_SIZE = 32765; // 32766-1
- public void enrichFields(final Map<String, Object> jsonObj, final
InputMarker inputMarker, final MetricData messageTruncateMetric) {
+ public Map<String, Object> enrichFields(final Map<String, Object> jsonObj,
final InputMarker inputMarker, final MetricData messageTruncateMetric) {
Input input = inputMarker.getInput();
// Update the block with the context fields
for (Map.Entry<String, String> entry :
input.getInputDescriptor().getAddFields().entrySet()) {
@@ -79,12 +80,17 @@ public void enrichFields(final Map<String, Object> jsonObj,
final InputMarker in
(Integer) inputMarker.getAllProperties().get("line_number") > 0) {
jsonObj.put("logfile_line_number",
inputMarker.getAllProperties().get("line_number"));
}
+ if (!jsonObj.containsKey("level")) {
+ jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN);
+ }
if (jsonObj.containsKey("log_message")) {
// TODO: Let's check size only for log_message for now
String logMessage = (String) jsonObj.get("log_message");
logMessage = truncateLongLogMessage(messageTruncateMetric, jsonObj,
input, logMessage);
jsonObj.put("message_md5", "" +
Hashing.md5().hashBytes(logMessage.getBytes()).asLong());
}
+
+ return jsonObj;
}
@SuppressWarnings("unchecked")
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
index b4c862d997..9f40b91f2e 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
@@ -82,10 +82,10 @@ public void init() throws Exception {
@SuppressWarnings("unchecked")
public void write(Map<String, Object> jsonObj, InputMarker inputMarker) {
jsonObj.put("seq_num", docCounter++);
- if (docCounter == Long.MIN_VALUE) {
+ if (docCounter == Long.MAX_VALUE) {
docCounter = 1;
}
- outputLineEnricher.enrichFields(jsonObj, inputMarker,
messageTruncateMetric);
+ jsonObj = outputLineEnricher.enrichFields(jsonObj, inputMarker,
messageTruncateMetric);
Input input = inputMarker.getInput();
List<String> defaultLogLevels = getDefaultLogLevels(input);
if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker, defaultLogLevels)
diff --git
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java
index 9be30a066e..4aca09dfa7 100644
---
a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java
+++
b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java
@@ -60,7 +60,7 @@
@Override
public void write(Map<String, Object> jsonObj, InputMarker marker) {
if (useFilters.get()) {
- outputLineEnricher.enrichFields(jsonObj, marker, messageTruncateMetric);
+ jsonObj = outputLineEnricher.enrichFields(jsonObj, marker,
messageTruncateMetric);
if (!outputLineFilter.apply(jsonObj, marker.getInput())) {
if (jsonObj.get("id") == null) {
jsonObj.put("id", IdGeneratorHelper.generateUUID(jsonObj,
storageOutput.getIdFields()));
diff --git
a/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json
b/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json
new file mode 100644
index 0000000000..863050e2d8
--- /dev/null
+++ b/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json
@@ -0,0 +1,67 @@
+{
+ "input": [
+ {
+ "type": "ams_grafana",
+ "rowtype": "service",
+ "path": "/root/test-logs/grafana/grafana.log"
+ }
+ ],
+ "filter": [
+ {
+ "filter": "grok",
+ "conditions": {
+ "fields": {
+ "type": [
+ "ams_grafana"
+ ]
+ }
+ },
+ "skipOnError": false,
+ "deepExtract": false,
+ "post_map_values": {
+ "level": [
+ {
+ "map_field_value": {
+ "pre_value": "I",
+ "post_value": "INFO"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "W",
+ "post_value": "WARN"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "D",
+ "post_value": "DEBUG"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "E",
+ "post_value": "ERROR"
+ }
+ },
+ {
+ "map_field_value": {
+ "pre_value": "F",
+ "post_value": "FATAL"
+ }
+ }
+ ],
+ "logtime": [
+ {
+ "map_date": {
+ "target_date_pattern": "yyyy/MM/dd HH:mm:ss"
+ }
+ }
+ ]
+ },
+ "log4j_format": "%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+ "multiline_pattern": "^(%{DATESTAMP:logtime})",
+ "message_pattern":
"(?m)^%{DATESTAMP:logtime}%{SPACE}(?:%{DATA:method})%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/docker/test-logs/grafana/grafana.log
b/docker/test-logs/grafana/grafana.log
new file mode 100644
index 0000000000..8d561b94db
--- /dev/null
+++ b/docker/test-logs/grafana/grafana.log
@@ -0,0 +1,8 @@
+2018/11/22 10:45:09 [I] Migrator: exec migration id: create index
UQE_quota_org_id_user_id_target - v1
+2018/11/22 10:45:09 [I] Created default admin user: admin
+2018/11/22 10:45:09 [I] Listen: http://0.0.0.0:3000
+Thu Nov 22 10:45:10 UTC 2018 pid_file has been written to
+OK
+2018/11/22 10:45:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E]
Could not find plugin definition for data source 1: ambari-metrics
+2018/11/22 10:46:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E]
Could not find plugin definition for data source 2: ambari-metrics
+2018/11/22 10:46:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E]
Could not find plugin definition for data source 3: ambari-metrics
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
