This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit d858fe017054c49c1e8e7bb18e8e6025d170fb31
Author: Yaguang Jia <jiayagu...@foxmail.com>
AuthorDate: Tue Mar 14 14:43:52 2023 +0800
KYLIN-5560 Add traceId into each HTTP request and into log4j MDC to help
tracing transactions.
---
build/conf/kylin-guardian-log4j.xml | 2 +-
build/conf/kylin-server-log4j.xml | 14 +++++++-------
build/conf/kylin-tools-diag-log4j.xml | 4 ++--
build/conf/kylin-tools-log4j.xml | 2 +-
build/conf/spark-appmaster-log4j.xml | 2 +-
build/conf/spark-driver-log4j.xml | 2 +-
build/conf/spark-executor-cloud-log4j.xml | 2 +-
build/conf/spark-streaming-driver-log4j.xml | 2 +-
src/common-booter/src/main/resources/log4j2.xml | 2 +-
.../java/org/apache/kylin/rest/interceptor/KEFilter.java | 11 +++++++++++
src/data-loading-booter/src/main/resources/log4j2.xml | 2 +-
src/query-booter/src/main/resources/log4j2.xml | 2 +-
src/server/src/main/resources/application.yaml | 4 ++--
src/server/src/main/resources/log4j2.xml | 2 +-
14 files changed, 32 insertions(+), 21 deletions(-)
diff --git a/build/conf/kylin-guardian-log4j.xml
b/build/conf/kylin-guardian-log4j.xml
index d488226bfb..681fda5ffe 100644
--- a/build/conf/kylin-guardian-log4j.xml
+++ b/build/conf/kylin-guardian-log4j.xml
@@ -25,7 +25,7 @@
</Policies>
<DefaultRolloverStrategy max="10"/>
</RollingFile>
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t] %c{2}
: %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Appenders>
<Loggers>
<Root level="INFO">
diff --git a/build/conf/kylin-server-log4j.xml
b/build/conf/kylin-server-log4j.xml
index 93e0b51299..14d22263ff 100644
--- a/build/conf/kylin-server-log4j.xml
+++ b/build/conf/kylin-server-log4j.xml
@@ -23,7 +23,7 @@
<SizeBasedTriggeringPolicy size="268435456"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</RollingRandomAccessFile>
<Routing name="routing">
<Routes pattern="$${ctx:logCategory}">
@@ -35,7 +35,7 @@
<SizeBasedTriggeringPolicy size="268435456"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- <PatternLayout pattern="%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</RollingFile>
</Route>
@@ -48,7 +48,7 @@
<SizeBasedTriggeringPolicy size="268435456"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</RollingFile>
<RollingFile name="spark-history-server"
fileName="${env:KYLIN_HOME}/logs/kylin.history_server.log"
append="true"
@@ -57,7 +57,7 @@
<SizeBasedTriggeringPolicy size="268435456"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</RollingFile>
<RollingFile name="build-log-spark"
fileName="${env:KYLIN_HOME}/logs/kylin.build.log" append="true"
filePattern="${env:KYLIN_HOME}/logs/kylin.build.log.%i">
@@ -65,7 +65,7 @@
<SizeBasedTriggeringPolicy size="268435456"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</RollingFile>
<RollingFile name="metadata-log-spark"
fileName="${env:KYLIN_HOME}/logs/kylin.metadata.log" append="true"
filePattern="${env:KYLIN_HOME}/logs/kylin.metadata.log.%i">
@@ -73,7 +73,7 @@
<SizeBasedTriggeringPolicy size="268435456"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</RollingFile>
</Appenders>
<Loggers>
@@ -110,7 +110,7 @@
<Logger name="org.apache.kylin.engine.spark.utils.LogEx" level="INFO"
/>
<Logger name="org.apache.kylin.rest.service.QueryCacheManager"
level="INFO"/>
<!-- Kerberos log -->
- <Logger name="io.kyligence.kap.tool.kerberos" level="INFO"/>
+ <Logger name="org.apache.kylin.tool.kerberos" level="INFO"/>
<!-- Other log -->
<Logger
name="org.apache.kylin.metadata.cube.storage.TotalStorageCollector"
level="INFO" />
<Logger name="org.apache.kylin.common.metrics.MetricsInfluxdbReporter"
level="INFO" />
diff --git a/build/conf/kylin-tools-diag-log4j.xml
b/build/conf/kylin-tools-diag-log4j.xml
index 11ab7976ec..728e50d2ca 100644
--- a/build/conf/kylin-tools-diag-log4j.xml
+++ b/build/conf/kylin-tools-diag-log4j.xml
@@ -23,10 +23,10 @@
<SizeBasedTriggeringPolicy size="10485760"/>
</Policies>
<DefaultRolloverStrategy max="1"/>
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</RollingFile>
<Console name="stdout" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
diff --git a/build/conf/kylin-tools-log4j.xml b/build/conf/kylin-tools-log4j.xml
index 5f605b7da5..e5e40d0beb 100644
--- a/build/conf/kylin-tools-log4j.xml
+++ b/build/conf/kylin-tools-log4j.xml
@@ -18,7 +18,7 @@
<Configuration name="Log4j1">
<Appenders>
<Console name="stderr" target="SYSTEM_ERR">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
diff --git a/build/conf/spark-appmaster-log4j.xml
b/build/conf/spark-appmaster-log4j.xml
index bd64793e07..fe02a00c27 100644
--- a/build/conf/spark-appmaster-log4j.xml
+++ b/build/conf/spark-appmaster-log4j.xml
@@ -18,7 +18,7 @@
<Configuration name="Log4j1">
<Appenders>
<Console name="stderr" target="SYSTEM_OUT"
layout="org.apache.spark.log4j.SensitivePatternLayout">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
diff --git a/build/conf/spark-driver-log4j.xml
b/build/conf/spark-driver-log4j.xml
index 04a771ae1f..7f827a1ef8 100644
--- a/build/conf/spark-driver-log4j.xml
+++ b/build/conf/spark-driver-log4j.xml
@@ -25,7 +25,7 @@
logPath="${sys:spark.driver.log4j.appender.hdfs.File}"
logQueueCapacity="5000"
flushInterval="5000">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</DriverHdfsAppender>
</Appenders>
<Loggers>
diff --git a/build/conf/spark-executor-cloud-log4j.xml
b/build/conf/spark-executor-cloud-log4j.xml
index f48a06ff14..a032caa2be 100644
--- a/build/conf/spark-executor-cloud-log4j.xml
+++ b/build/conf/spark-executor-cloud-log4j.xml
@@ -18,7 +18,7 @@
<Configuration name="Log4j1">
<Appenders>
<Console name="stderr" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
diff --git a/build/conf/spark-streaming-driver-log4j.xml
b/build/conf/spark-streaming-driver-log4j.xml
index ab9e0d744c..636212c1cc 100644
--- a/build/conf/spark-streaming-driver-log4j.xml
+++ b/build/conf/spark-streaming-driver-log4j.xml
@@ -26,7 +26,7 @@
rollingByteSize="${sys:kap.spark.log4j.appender.rolling.size}"
logQueueCapacity="5000"
flushInterval="5000">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</DriverHdfsRollingAppender>
</Appenders>
<Loggers>
diff --git a/src/common-booter/src/main/resources/log4j2.xml
b/src/common-booter/src/main/resources/log4j2.xml
index aefc89d83d..486437182e 100644
--- a/src/common-booter/src/main/resources/log4j2.xml
+++ b/src/common-booter/src/main/resources/log4j2.xml
@@ -18,7 +18,7 @@
<Configuration name="Log4j1">
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
diff --git
a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
index 0e238cec4f..071a4ab619 100644
---
a/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
+++
b/src/common-service/src/main/java/org/apache/kylin/rest/interceptor/KEFilter.java
@@ -33,6 +33,8 @@ import org.apache.kylin.common.exception.ExceptionResolve;
import org.apache.kylin.common.exception.code.ErrorMsg;
import org.apache.kylin.common.exception.code.ErrorSuggestion;
import org.apache.kylin.common.msg.MsgPicker;
+import org.apache.kylin.common.util.RandomUtil;
+import org.apache.logging.log4j.ThreadContext;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
@@ -59,7 +61,16 @@ public class KEFilter extends OncePerRequestFilter {
QueryContext.currentTrace().startSpan(QueryTrace.HTTP_RECEPTION);
}
+ // Set traceId for KE
+ String traceId = RandomUtil.randomUUIDStr();
+ ThreadContext.put("traceId", String.format("traceId: %s ", traceId));
+
filterChain.doFilter(request, response);
+
+ // clean ThreadContext
+ ThreadContext.clearAll();
+ // set `traceId` attribute for accesslog
+ request.setAttribute("traceId", traceId);
}
}
diff --git a/src/data-loading-booter/src/main/resources/log4j2.xml
b/src/data-loading-booter/src/main/resources/log4j2.xml
index aefc89d83d..486437182e 100644
--- a/src/data-loading-booter/src/main/resources/log4j2.xml
+++ b/src/data-loading-booter/src/main/resources/log4j2.xml
@@ -18,7 +18,7 @@
<Configuration name="Log4j1">
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
diff --git a/src/query-booter/src/main/resources/log4j2.xml
b/src/query-booter/src/main/resources/log4j2.xml
index 61db473095..cd292d4519 100644
--- a/src/query-booter/src/main/resources/log4j2.xml
+++ b/src/query-booter/src/main/resources/log4j2.xml
@@ -18,7 +18,7 @@
<Configuration name="Log4j1">
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
diff --git a/src/server/src/main/resources/application.yaml
b/src/server/src/main/resources/application.yaml
index 151df227e8..edd4fe5530 100644
--- a/src/server/src/main/resources/application.yaml
+++ b/src/server/src/main/resources/application.yaml
@@ -21,7 +21,7 @@ server:
max-http-form-post-size: 10485760
accesslog:
enabled: true
- pattern: '%h %l %u %{User-Agent}i %t "%r" %s %b %D'
+ pattern: 'traceId: %{traceId}r %h %l %u %{User-Agent}i %t "%r" %s %b %D'
max-days: ${kylin.server.accesslog-max-days:10}
compression:
enabled: true
@@ -130,4 +130,4 @@ spring:
import: optional:${KYLIN_CONF}/kylin.properties
web:
resources:
- static-locations: "file://${KYLIN_HOME}/kystudio/dist"
\ No newline at end of file
+ static-locations: "file://${KYLIN_HOME}/kystudio/dist"
diff --git a/src/server/src/main/resources/log4j2.xml
b/src/server/src/main/resources/log4j2.xml
index 8afee65ca2..4f3e36988e 100644
--- a/src/server/src/main/resources/log4j2.xml
+++ b/src/server/src/main/resources/log4j2.xml
@@ -18,7 +18,7 @@
<Configuration name="Log4j1">
<Appenders>
<Console name="stdout" target="SYSTEM_OUT">
- <PatternLayout pattern="%d{ISO8601} %-5p %X{request.project}[%t]
%c{2} : %mask{%m}%n"/>
+ <PatternLayout pattern="%X{traceId}%d{ISO8601} %-5p
%X{request.project}[%t] %c{2} : %mask{%m}%n"/>
</Console>
</Appenders>
<Loggers>
