This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new b2db89c AMBARI-24143 - Logs not indexed with default configuration
for many components when custom service accounts are used (#1579)
b2db89c is described below
commit b2db89c3800caf0248ea61a392a0671201bcf26f
Author: kasakrisz <[email protected]>
AuthorDate: Wed Jun 20 15:02:17 2018 +0200
AMBARI-24143 - Logs not indexed with default configuration for many
components when custom service accounts are used (#1579)
---
.../logsearch/patterns/AmbariLogPatternIT.java | 24 ++++++++-
.../logsearch/patterns/HiveLogPatterntIT.java | 12 ++---
.../logsearch/patterns/YarnLogPatternIT.java | 6 +--
.../0.5.0/configuration/logfeeder-env.xml | 22 ++++++++
.../LOGSEARCH/0.5.0/package/scripts/params.py | 5 +-
.../0.5.0/properties/input.config-ambari.json.j2 | 62 ++++++++++++++++++----
6 files changed, 107 insertions(+), 24 deletions(-)
diff --git
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java
index 34f17c7..7397df0 100644
---
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java
+++
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java
@@ -179,13 +179,33 @@ public class AmbariLogPatternIT extends PatternITBase {
}
@Test
- public void testAmbariEclipseLinkEntry() throws Exception {
+ public void testAmbariEclipseLinkSevereEntry() throws Exception {
+ testAmbariEclipseLinkEntry("Severe", "ERROR");
+ }
+
+ @Test
+ public void testAmbariEclipseLinkWarningEntry() throws Exception {
+ testAmbariEclipseLinkEntry("Warning", "WARN");
+ }
+
+ @Test
+ public void testAmbariEclipseLinkInfoEntry() throws Exception {
+ testAmbariEclipseLinkEntry("Info", "INFO");
+ }
+
+ @Test
+ public void testAmbariEclipseLinkConfigEntry() throws Exception {
+ testAmbariEclipseLinkEntry("Config", "INFO");
+ }
+
+ private void testAmbariEclipseLinkEntry(String logLevel, String
expectedLogLevel) throws Exception {
// given
- String logEntry = "[EL Info]: 2018-05-02
09:27:17.79--ServerSession(1657512321)-- EclipseLink, version: Eclipse
Persistence Services - 2.6.2.v20151217-774c696";
+ String logEntry = "[EL " + logLevel + "]: 2018-05-02
09:27:17.79--ServerSession(1657512321)-- EclipseLink, version: Eclipse
Persistence Services - 2.6.2.v20151217-774c696";
// when
Map<String, Object> result = testLogEntry(logEntry, "ambari_eclipselink",
ambariInputConfigTemplate());
// then
assertThat(result.isEmpty(), is(false));
+ assertThat(result.get("level"), is(expectedLogLevel));
assertThat(result.get("cluster"), is(CLUSTER));
assertThat(result.get("event_count"), is(1));
assertThat(result.get("type"), is("ambari_eclipselink"));
diff --git
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
index 988c758..1b4f18d 100644
---
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
+++
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java
@@ -33,16 +33,16 @@ import org.junit.Test;
public class HiveLogPatterntIT extends PatternITBase {
@Test
- public void testHiveServer2LogEntry() throws Exception {
+ public void testHiveServerLogEntry() throws Exception {
String logEntry = "2018-05-11T07:46:01,087 WARN [main]:
metastore.HiveMetaStoreClient (:()) - Failed to connect to the MetaStore
Server...";
- Map<String, Object> result = testLogEntry(logEntry,"hive_hiveserver2",
inputConfigTemplate(
+ Map<String, Object> result = testLogEntry(logEntry,"hive_server",
inputConfigTemplate(
new File(HDP_SERVICES_FOLDER,
"HIVE/package/templates/input.config-hive.json.j2")));
assertThat(result.isEmpty(), is(false));
assertThat(result.get("cluster"), is(CLUSTER));
assertThat(result.get("level"), is("WARN"));
assertThat(result.get("event_count"), is(1));
- assertThat(result.get("type"), is("hive_hiveserver2"));
+ assertThat(result.get("type"), is("hive_server"));
assertThat(result.containsKey("seq_num"), is(true));
assertThat(result.containsKey("id"), is(true));
assertThat(result.containsKey("message_md5"), is(true));
@@ -58,16 +58,16 @@ public class HiveLogPatterntIT extends PatternITBase {
}
@Test
- public void testHiveServer2InteractiveLogEntry() throws Exception {
+ public void testHiveServerInteractiveLogEntry() throws Exception {
String logEntry = "2018-05-11T08:48:02,973 WARN [main]: conf.HiveConf
(HiveConf.java:initialize(5193)) - HiveConf of name
hive.hook.proto.base-directory does not exist";
- Map<String, Object> result = testLogEntry(logEntry,"hive_hiveserver2",
inputConfigTemplate(
+ Map<String, Object> result =
testLogEntry(logEntry,"hive_server_interactive", inputConfigTemplate(
new File(HDP_SERVICES_FOLDER,
"HIVE/package/templates/input.config-hive.json.j2")));
assertThat(result.isEmpty(), is(false));
assertThat(result.get("cluster"), is(CLUSTER));
assertThat(result.get("level"), is("WARN"));
assertThat(result.get("event_count"), is(1));
- assertThat(result.get("type"), is("hive_hiveserver2"));
+ assertThat(result.get("type"), is("hive_server_interactive"));
assertThat(result.containsKey("seq_num"), is(true));
assertThat(result.containsKey("id"), is(true));
assertThat(result.containsKey("message_md5"), is(true));
diff --git
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
index 4fafd88..3a0f766 100644
---
a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
+++
b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java
@@ -133,14 +133,14 @@ public class YarnLogPatternIT extends PatternITBase {
@Test
public void testYarnHistoryServerLogEntry() throws Exception {
Map<String, Object> result = testLogEntry("2018-05-02 10:02:54,215 INFO
webapp.View (HsJobsBlock.java:render(74)) - Getting list of all Jobs.",
- "yarn_historyserver",
- inputConfigTemplate(new File(HDP_SERVICES_FOLDER,
"YARN/package/templates/input.config-yarn.json.j2")));
+ "mapred_historyserver",
+ inputConfigTemplate(new File(HDP_SERVICES_FOLDER,
"YARN/package/templates/input.config-mapreduce2.json.j2")));
assertThat(result.isEmpty(), is(false));
assertThat(result.get("cluster"), is(CLUSTER));
assertThat(result.get("level"), is("INFO"));
assertThat(result.get("event_count"), is(1));
- assertThat(result.get("type"), is("yarn_historyserver"));
+ assertThat(result.get("type"), is("mapred_historyserver"));
assertThat(result.get("logger_name"), is("webapp.View "));
assertThat(result.containsKey("seq_num"), is(true));
assertThat(result.containsKey("id"), is(true));
diff --git
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
index 4afd7b6..3076a06 100644
---
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
+++
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-env.xml
@@ -151,6 +151,28 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>ambari_server_log_dir</name>
+ <value>/var/log/ambari-server</value>
+ <description>Log dir of Ambari Server. Use this property when
bootstrapping Logsearch only.</description>
+ <display-name>Ambari Server log dir</display-name>
+ <value-attributes>
+ <type>directory</type>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>ambari_agent_log_dir</name>
+ <value>/var/log/ambari-agent</value>
+ <description>Log dir of Ambari Agent. Use this property when bootstrapping
Logsearch only.</description>
+ <display-name>Ambari Agent log dir</display-name>
+ <value-attributes>
+ <type>directory</type>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
<name>content</name>
<display-name>logfeeder-env template</display-name>
<description>This is the jinja template for logfeeder-env.sh
file</description>
diff --git
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 8194287..a492175 100644
---
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -25,6 +25,7 @@ from resource_management.libraries.functions.default import
default
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.script.script import Script
+
import status_params
@@ -218,8 +219,8 @@ logsearch_audit_logs_solrconfig_content =
config['configurations']['logsearch-au
logsearch_app_log4j_content =
config['configurations']['logsearch-log4j']['content']
# Log dirs
-ambari_server_log_dir = '/var/log/ambari-server'
-ambari_agent_log_dir = '/var/log/ambari-agent'
+ambari_server_log_dir =
default('/configurations/logfeeder-env/ambari_server_log_dir',
"/var/log/ambari-server")
+ambari_agent_log_dir =
default('/configurations/logfeeder-env/ambari_agent_log_dir',
"/var/log/ambari-agent")
# System logs
logfeeder_system_messages_content =
config['configurations']['logfeeder-system_log-env']['logfeeder_system_messages_content']
diff --git
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2
index 237740a..ea08e17 100644
---
a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2
+++
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2
@@ -184,25 +184,65 @@
},
"log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n",
- "multiline_pattern":"^(\\[EL%{SPACE}%{LOGLEVEL:level}\\])",
-
"message_pattern":"(?m)^\\[EL%{SPACE}%{LOGLEVEL:level}\\]:%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{GREEDYDATA:log_message}",
+ "multiline_pattern":"^(\\[EL%{SPACE}%{DATA:level}\\])",
+
"message_pattern":"(?m)^\\[EL%{SPACE}%{DATA:level}\\]:%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{GREEDYDATA:log_message}",
"post_map_values":{
"logtime":{
"map_date":{
"target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
}
-
},
- "level":{
- "map_fieldvalue":{
- "pre_value":"Warning",
- "post_value":"Warn"
+ "level":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"Severe",
+ "post_value":"ERROR"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Warning",
+ "post_value":"WARN"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Finer",
+ "post_value":"WARN"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Info",
+ "post_value":"INFO"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Config",
+ "post_value":"INFO"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Fine",
+ "post_value":"DEBUG"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Finest",
+ "post_value":"TRACE"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"All",
+ "post_value":"TRACE"
+ }
}
-
- }
-
+ ]
}
-
},
{
"filter":"grok",
