This is an automated email from the ASF dual-hosted git repository.
gyfora pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new 9099abd2 [FLINK-39501] Logback Support in Flink Kubernetes Operator
9099abd2 is described below
commit 9099abd27f2a9de20e79fd2e194f2a593b26dee2
Author: Dennis-Mircea Ciupitu <[email protected]>
AuthorDate: Tue Apr 28 17:12:33 2026 +0300
[FLINK-39501] Logback Support in Flink Kubernetes Operator
---
.github/workflows/ci.yml | 21 ++++
.github/workflows/e2e.yaml | 3 +
Dockerfile | 5 +-
docker-entrypoint.sh | 14 ++-
docs/content.zh/docs/operations/helm.md | 3 +
docs/content.zh/docs/operations/metrics-logging.md | 101 ++++++++++++++++--
docs/content/docs/operations/helm.md | 3 +
docs/content/docs/operations/metrics-logging.md | 101 ++++++++++++++++--
e2e-tests/test_logback_logging.sh | 116 +++++++++++++++++++++
flink-kubernetes-operator/pom.xml | 34 +++++-
.../conf/logback-console.xml | 56 ++++++++++
.../conf/logback-operator.xml | 35 +++++++
.../templates/_helpers.tpl | 12 +++
.../templates/controller/configmap.yaml | 12 +++
.../templates/controller/deployment.yaml | 15 ++-
.../tests/controller/configmap_test.yaml | 32 ++++++
.../tests/controller/deployment_test.yaml | 80 ++++++++++++++
helm/flink-kubernetes-operator/values.yaml | 8 +-
pom.xml | 1 +
19 files changed, 631 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e11507ef..1dd5a143 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -235,3 +235,24 @@ jobs:
flink-version: ${{ matrix.flink-version }}
test: ${{ matrix.test }}
mode: ${{ matrix.mode }}
+ e2e_logging:
+ name: Logging framework tests
+ needs: e2e_smoke_test
+ strategy:
+ matrix:
+ flink-version:
+ - "v2_2"
+ - "v2_1"
+ - "v2_0"
+ - "v1_20"
+ mode:
+ - "native"
+ - "standalone"
+ test:
+ - test_logback_logging.sh
+ uses: ./.github/workflows/e2e.yaml
+ with:
+ java-version: 17
+ flink-version: ${{ matrix.flink-version }}
+ test: ${{ matrix.test }}
+ mode: ${{ matrix.mode }}
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 5434da7a..02b7bd3d 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -78,6 +78,9 @@ jobs:
if [[ "${{ inputs.test }}" == "test_dynamic_config.sh" ]]; then
sed -i "s/flink-conf.yaml: |+/config.yaml: |+/"
helm/flink-kubernetes-operator/values.yaml
fi
+ if [[ "${{ inputs.test }}" == "test_logback_logging.sh" ]]; then
+ sed -i 's/framework: "log4j2"/framework: "logback"/'
helm/flink-kubernetes-operator/values.yaml
+ fi
helm --debug install flink-kubernetes-operator -n ${{
inputs.namespace }} helm/flink-kubernetes-operator --set
image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{
steps.namespace-creator.outputs.EXTRA_HELM_INSTALL_ARGS }}
kubectl wait --for=condition=Available --timeout=120s -n ${{
inputs.namespace }} deploy/flink-kubernetes-operator
kubectl get pods -n ${{ inputs.namespace }}
diff --git a/Dockerfile b/Dockerfile
index c80d0686..2f0bfc39 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -44,7 +44,8 @@ ENV
WEBHOOK_JAR=flink-kubernetes-webhook-$OPERATOR_VERSION-shaded.jar
ENV KUBERNETES_STANDALONE_JAR=flink-kubernetes-standalone-$OPERATOR_VERSION.jar
ENV OPERATOR_LIB=$FLINK_HOME/operator-lib
-RUN mkdir -p $OPERATOR_LIB
+ENV OPERATOR_LOG=$FLINK_HOME/log
+RUN mkdir -p $OPERATOR_LIB $OPERATOR_LOG/log4j $OPERATOR_LOG/logback
WORKDIR /flink-kubernetes-operator
RUN groupadd --system --gid=9999 flink && \
@@ -56,6 +57,8 @@ COPY --chown=flink:flink --from=build
/app/flink-kubernetes-operator/target/$OPE
COPY --chown=flink:flink --from=build
/app/flink-kubernetes-webhook/target/$WEBHOOK_JAR .
COPY --chown=flink:flink --from=build
/app/flink-kubernetes-standalone/target/$KUBERNETES_STANDALONE_JAR .
COPY --chown=flink:flink --from=build
/app/flink-kubernetes-operator/target/plugins $FLINK_HOME/plugins
+COPY --chown=flink:flink --from=build
/app/flink-kubernetes-operator/target/log4j/ $FLINK_HOME/log/log4j/
+COPY --chown=flink:flink --from=build
/app/flink-kubernetes-operator/target/logback/ $FLINK_HOME/log/logback/
COPY --chown=flink:flink --from=build
/app/tools/license/licenses-output/NOTICE .
COPY --chown=flink:flink --from=build
/app/tools/license/licenses-output/licenses ./licenses
COPY --chown=flink:flink --from=build /app/LICENSE ./LICENSE
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index 7464645e..db1dde61 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -43,6 +43,16 @@ maybe_enable_jemalloc() {
maybe_enable_jemalloc
+# Select the logging framework JARs to put on the classpath.
+# Supported values: "log4j2" (default), "logback".
+OPERATOR_LOG=${FLINK_HOME:-/opt/flink}/log
+LOGGING_FRAMEWORK="${LOGGING_FRAMEWORK:-log4j2}"
+if [ "$LOGGING_FRAMEWORK" = "logback" ]; then
+ LOG_CLASSPATH="$OPERATOR_LOG/logback/*"
+else
+ LOG_CLASSPATH="$OPERATOR_LOG/log4j/*"
+fi
+
if [ "$1" = "help" ]; then
printf "Usage: $(basename "$0") (operator|webhook)\n"
printf " Or $(basename "$0") help\n\n"
@@ -50,12 +60,12 @@ if [ "$1" = "help" ]; then
elif [ "$1" = "operator" ]; then
echo "Starting Operator"
- exec java -cp
"./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:$OPERATOR_LIB/*" $LOG_CONFIG
$JVM_ARGS org.apache.flink.kubernetes.operator.FlinkOperator
+ exec java -cp
"./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:$LOG_CLASSPATH:$OPERATOR_LIB/*"
$LOG_CONFIG $JVM_ARGS org.apache.flink.kubernetes.operator.FlinkOperator
elif [ "$1" = "webhook" ]; then
echo "Starting Webhook"
# Adds the operator shaded jar on the classpath when the webhook starts
- exec java -cp
"./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:./$WEBHOOK_JAR:$OPERATOR_LIB/*"
$LOG_CONFIG $JVM_ARGS
org.apache.flink.kubernetes.operator.admission.FlinkOperatorWebhook
+ exec java -cp
"./$KUBERNETES_STANDALONE_JAR:./$OPERATOR_JAR:./$WEBHOOK_JAR:$LOG_CLASSPATH:$OPERATOR_LIB/*"
$LOG_CONFIG $JVM_ARGS
org.apache.flink.kubernetes.operator.admission.FlinkOperatorWebhook
fi
args=("${args[@]}")
diff --git a/docs/content.zh/docs/operations/helm.md
b/docs/content.zh/docs/operations/helm.md
index b2a843a6..704a3f10 100644
--- a/docs/content.zh/docs/operations/helm.md
+++ b/docs/content.zh/docs/operations/helm.md
@@ -76,6 +76,8 @@ The configurable parameters of the Helm chart and which
default values as detail
| defaultConfiguration.flink-conf.yaml | The default configuration
of flink-conf.yaml.
|
kubernetes.operator.metrics.reporter.slf4j.factory.class:
org.apache.flink.metrics.slf4j.Slf4jReporterFactory<br/>kubernetes.operator.metrics.reporter.slf4j.interval:
5 MINUTE<br/>kubernetes.operator.reconcile.interval: 15
s<br/>kubernetes.operator.observer.progress-check.interva [...]
| defaultConfiguration.log4j-console.properties | The default configuration
of log4j-console.properties.
|
[...]
| defaultConfiguration.log4j-operator.properties | The default configuration
of log4j-operator.properties.
|
[...]
+| defaultConfiguration.logback-operator.xml | The default configuration
of logback-operator.xml. Used when `logging.framework` is set to `logback`.
|
[...]
+| defaultConfiguration.logback-console.xml | The default configuration
of logback-console.xml. Used when `logging.framework` is set to `logback`.
|
[...]
| fullnameOverride | Overrides the fullname with
the specified full name.
|
[...]
| image.digest | The image tag of
flink-kubernetes-operator. If set then it takes precedence and the image tag
will be ignored. |
[...]
| image.pullPolicy | The image pull policy of
flink-kubernetes-operator.
| IfNotPresent
[...]
@@ -87,6 +89,7 @@ The configurable parameters of the Helm chart and which
default values as detail
| jobServiceAccount.name | The name of job service
account.
| flink
[...]
| jvmArgs.operator | The JVM start up options
for operator.
|
[...]
| jvmArgs.webhook | The JVM start up options
for webhook.
|
[...]
+| logging.framework | The logging framework to
use for the operator. Supported values: `log4j2`, `logback`. Controls which
config files are mounted and which JVM flag is passed. | log4j2
[...]
| metrics.port | The metrics port on the
container for default configuration.
|
[...]
| nameOverride | Overrides the name with the
specified name.
|
[...]
| operatorHealth.livenessProbe | Liveness probe
configuration for the operator using the health endpoint. Only time settings
should be configured, endpoint is set automatically based on port. |
[...]
diff --git a/docs/content.zh/docs/operations/metrics-logging.md
b/docs/content.zh/docs/operations/metrics-logging.md
index d1a585dd..af7afb6d 100644
--- a/docs/content.zh/docs/operations/metrics-logging.md
+++ b/docs/content.zh/docs/operations/metrics-logging.md
@@ -616,7 +616,33 @@ Once the custom resource is created in the Kubernetes
environment the operator m
## Logging
The Operator controls the logging behaviour for Flink applications and the
Operator itself using configuration files mounted externally via ConfigMaps.
[Configuration
files](https://github.com/apache/flink-kubernetes-operator/tree/main/helm/flink-kubernetes-operator/conf)
with default values are shipped in the Helm chart. It is recommended to review
and adjust them if needed in the `values.yaml` file before deploying the
Operator in production environments.
-To append/override the default log configuration properties for the operator
and Flink deployments define the `log4j-operator.properties` and
`log4j-console.properties` keys respectively:
+The operator supports two logging frameworks: **Log4j2** (default) and
**Logback**. Both sets of configuration files are shipped in the Helm chart and
Docker image. The active framework is selected at install time via the
`logging.framework` Helm value, either in `values.yaml` or on the command line:
+
+```shell
+# Use the default Log4j2 framework (explicit, same as omitting the flag)
+helm install flink-operator helm/flink-kubernetes-operator --set
logging.framework=log4j2
+
+# Switch to Logback
+helm install flink-operator helm/flink-kubernetes-operator --set
logging.framework=logback
+```
+
+This controls three things:
+1. Which logging configuration files are mounted into `/opt/flink/conf/`
+2. Which JVM system property is passed to select the framework
(`log4j.configurationFile` or `logback.configurationFile`)
+3. Which SLF4J binding JAR is placed on the classpath at runtime
+
+{{< hint info >}}
+Logging in the operator is intentionally succinct and does not include
contextual information such as namespace or name of the FlinkDeployment objects.
+We rely on the MDC provided by the operator-sdk to access this information and
use it directly in the log layout.
+
+See the [Java Operator SDK
docs](https://javaoperatorsdk.io/docs/features#contextual-info-for-logging-with-mdc)
for more detail.
+{{< /hint >}}
+
+To learn more about accessing the job logs or changing the log level
dynamically check the corresponding
[section](https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/resource-providers/native_kubernetes/#logging)
of the core documentation.
+
+### Log4j2
+
+Log4j2 is the default logging framework. To append/override the default log
configuration properties for the operator and Flink deployments define the
`log4j-operator.properties` and `log4j-console.properties` keys respectively:
```yaml
defaultConfiguration:
@@ -634,18 +660,60 @@ defaultConfiguration:
# monitorInterval = 30
```
-{{< hint info >}}
-Logging in the operator is intentionally succinct and does not include
contextual information such as namespace or name of the FlinkDeployment objects.
-We rely on the MDC provided by the operator-sdk to access this information and
use it directly in the log layout.
+### Logback
-See the [Java Operator SDK
docs](https://javaoperatorsdk.io/docs/features#contextual-info-for-logging-with-mdc)
for more detail.
+When using Logback, the equivalent configuration keys are
`logback-operator.xml` and `logback-console.xml`:
+
+```yaml
+logging:
+ framework: logback
+
+defaultConfiguration:
+ create: true
+ append: true
+ logback-operator.xml: |+
+ <!-- Flink Operator Logback Overrides -->
+ <!-- <configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %-60logger{26} %X
- %msg%n</pattern>
+ </encoder>
+ </appender>
+ <root level="DEBUG">
+ <appender-ref ref="ConsoleAppender" />
+ </root>
+ </configuration> -->
+ logback-console.xml: |+
+ <!-- Flink Deployment Logback Overrides -->
+ <!-- <configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %-60logger{26} %X
- %msg%n</pattern>
+ </encoder>
+ </appender>
+ <root level="DEBUG">
+ <appender-ref ref="ConsoleAppender" />
+ </root>
+ </configuration> -->
+```
+
+{{< hint warning >}}
+**Logback XML overrides replace the entire default configuration.** Unlike
Log4j2 `.properties` files where user entries are appended, XML files cannot be
concatenated (two `<configuration>` root elements produce invalid XML). Ensure
your custom `logback-operator.xml` or `logback-console.xml` is complete and
self-contained. Logback supports only XML-based configuration and does not
provide native support for `.properties` configuration files.
{{< /hint >}}
-To learn more about accessing the job logs or changing the log level
dynamically check the corresponding
[section](https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/resource-providers/native_kubernetes/#logging)
of the core documentation.
+### Logging Library Version Overrides
+
+The operator ships with **Logback 1.2.x** and **SLF4J 1.7.x**. These versions
are bundled in the Docker image and the SLF4J 1.7.x API is shaded into the
operator JAR.
+
+{{< hint warning >}}
+**Upgrading to Logback 1.4+/1.5+ or SLF4J 2.x is not supported.** SLF4J 2.x
uses a `ServiceLoader`-based binding mechanism that is incompatible with the
SLF4J 1.7.x API shaded inside the operator. Replacing the JARs at runtime will
result in `ClassNotFoundException: org.slf4j.impl.StaticLoggerBinder`.
+{{< /hint >}}
### FlinkDeployment Logging Configuration
-Users have the freedom to override the default `log4j-console.properties`
settings on a per-deployment level by putting the entire log configuration into
`spec.logConfiguration`:
+Users have the freedom to override the default logging settings on a
per-deployment level by putting the entire log configuration into
`spec.logConfiguration`.
+
+For Log4j2:
```yaml
spec:
@@ -656,3 +724,22 @@ spec:
rootLogger.appenderRef.file.ref = LogFile
...
```
+
+For Logback:
+
+```yaml
+spec:
+ ...
+ logConfiguration:
+ logback-console.xml: |+
+ <configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %-5level %-60.60logger{60} %X
- %msg%n</pattern>
+ </encoder>
+ </appender>
+ <root level="DEBUG">
+ <appender-ref ref="ConsoleAppender" />
+ </root>
+ </configuration>
+```
diff --git a/docs/content/docs/operations/helm.md
b/docs/content/docs/operations/helm.md
index 84e14828..64d98f2c 100644
--- a/docs/content/docs/operations/helm.md
+++ b/docs/content/docs/operations/helm.md
@@ -76,6 +76,8 @@ The configurable parameters of the Helm chart and which
default values as detail
| defaultConfiguration.flink-conf.yaml | The default configuration
of flink-conf.yaml.
|
kubernetes.operator.metrics.reporter.slf4j.factory.class:
org.apache.flink.metrics.slf4j.Slf4jReporterFactory<br/>kubernetes.operator.metrics.reporter.slf4j.interval:
5 MINUTE<br/>kubernetes.operator.reconcile.interval: 15
s<br/>kubernetes.operator.observer.progress-check.interva [...]
| defaultConfiguration.log4j-console.properties | The default configuration
of log4j-console.properties.
|
[...]
| defaultConfiguration.log4j-operator.properties | The default configuration
of log4j-operator.properties.
|
[...]
+| defaultConfiguration.logback-operator.xml | The default configuration
of logback-operator.xml. Used when `logging.framework` is set to `logback`.
|
[...]
+| defaultConfiguration.logback-console.xml | The default configuration
of logback-console.xml. Used when `logging.framework` is set to `logback`.
|
[...]
| fullnameOverride | Overrides the fullname with
the specified full name.
|
[...]
| image.digest | The image tag of
flink-kubernetes-operator. If set then it takes precedence and the image tag
will be ignored. |
[...]
| image.pullPolicy | The image pull policy of
flink-kubernetes-operator.
| IfNotPresent
[...]
@@ -87,6 +89,7 @@ The configurable parameters of the Helm chart and which
default values as detail
| jobServiceAccount.name | The name of job service
account.
| flink
[...]
| jvmArgs.operator | The JVM start up options
for operator.
|
[...]
| jvmArgs.webhook | The JVM start up options
for webhook.
|
[...]
+| logging.framework | The logging framework to
use for the operator. Supported values: `log4j2`, `logback`. Controls which
config files are mounted and which JVM flag is passed. | log4j2
[...]
| metrics.port | The metrics port on the
container for default configuration.
|
[...]
| nameOverride | Overrides the name with the
specified name.
|
[...]
| operatorHealth.livenessProbe | Liveness probe
configuration for the operator using the health endpoint. Only time settings
should be configured, endpoint is set automatically based on port. |
[...]
diff --git a/docs/content/docs/operations/metrics-logging.md
b/docs/content/docs/operations/metrics-logging.md
index dfe08ab6..f4451d72 100644
--- a/docs/content/docs/operations/metrics-logging.md
+++ b/docs/content/docs/operations/metrics-logging.md
@@ -616,7 +616,33 @@ Once the custom resource is created in the Kubernetes
environment the operator m
## Logging
The Operator controls the logging behaviour for Flink applications and the
Operator itself using configuration files mounted externally via ConfigMaps.
[Configuration
files](https://github.com/apache/flink-kubernetes-operator/tree/main/helm/flink-kubernetes-operator/conf)
with default values are shipped in the Helm chart. It is recommended to review
and adjust them if needed in the `values.yaml` file before deploying the
Operator in production environments.
-To append/override the default log configuration properties for the operator
and Flink deployments define the `log4j-operator.properties` and
`log4j-console.properties` keys respectively:
+The operator supports two logging frameworks: **Log4j2** (default) and
**Logback**. Both sets of configuration files are shipped in the Helm chart and
Docker image. The active framework is selected at install time via the
`logging.framework` Helm value, either in `values.yaml` or on the command line:
+
+```shell
+# Use the default Log4j2 framework (explicit, same as omitting the flag)
+helm install flink-operator helm/flink-kubernetes-operator --set
logging.framework=log4j2
+
+# Switch to Logback
+helm install flink-operator helm/flink-kubernetes-operator --set
logging.framework=logback
+```
+
+This controls three things:
+1. Which logging configuration files are mounted into `/opt/flink/conf/`
+2. Which JVM system property is passed to select the framework
(`log4j.configurationFile` or `logback.configurationFile`)
+3. Which SLF4J binding JAR is placed on the classpath at runtime
+
+{{< hint info >}}
+Logging in the operator is intentionally succinct and does not include
contextual information such as namespace or name of the FlinkDeployment objects.
+We rely on the MDC provided by the operator-sdk to access this information and
use it directly in the log layout.
+
+See the [Java Operator SDK
docs](https://javaoperatorsdk.io/docs/features#contextual-info-for-logging-with-mdc)
for more detail.
+{{< /hint >}}
+
+To learn more about accessing the job logs or changing the log level
dynamically check the corresponding
[section](https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/resource-providers/native_kubernetes/#logging)
of the core documentation.
+
+### Log4j2
+
+Log4j2 is the default logging framework. To append/override the default log
configuration properties for the operator and Flink deployments define the
`log4j-operator.properties` and `log4j-console.properties` keys respectively:
```yaml
defaultConfiguration:
@@ -634,18 +660,60 @@ defaultConfiguration:
# monitorInterval = 30
```
-{{< hint info >}}
-Logging in the operator is intentionally succinct and does not include
contextual information such as namespace or name of the FlinkDeployment objects.
-We rely on the MDC provided by the operator-sdk to access this information and
use it directly in the log layout.
+### Logback
-See the [Java Operator SDK
docs](https://javaoperatorsdk.io/docs/features#contextual-info-for-logging-with-mdc)
for more detail.
+When using Logback, the equivalent configuration keys are
`logback-operator.xml` and `logback-console.xml`:
+
+```yaml
+logging:
+ framework: logback
+
+defaultConfiguration:
+ create: true
+ append: true
+ logback-operator.xml: |+
+ <!-- Flink Operator Logback Overrides -->
+ <!-- <configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %-60logger{26} %X
- %msg%n</pattern>
+ </encoder>
+ </appender>
+ <root level="DEBUG">
+ <appender-ref ref="ConsoleAppender" />
+ </root>
+ </configuration> -->
+ logback-console.xml: |+
+ <!-- Flink Deployment Logback Overrides -->
+ <!-- <configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %-60logger{26} %X
- %msg%n</pattern>
+ </encoder>
+ </appender>
+ <root level="DEBUG">
+ <appender-ref ref="ConsoleAppender" />
+ </root>
+ </configuration> -->
+```
+
+{{< hint warning >}}
+**Logback XML overrides replace the entire default configuration.** Unlike
Log4j2 `.properties` files where user entries are appended, XML files cannot be
concatenated (two `<configuration>` root elements produce invalid XML). Ensure
your custom `logback-operator.xml` or `logback-console.xml` is complete and
self-contained. Logback supports only XML-based configuration and does not
provide native support for `.properties` configuration files.
{{< /hint >}}
-To learn more about accessing the job logs or changing the log level
dynamically check the corresponding
[section](https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/resource-providers/native_kubernetes/#logging)
of the core documentation.
+### Logging Library Version Overrides
+
+The operator ships with **Logback 1.2.x** and **SLF4J 1.7.x**. These versions
are bundled in the Docker image and the SLF4J 1.7.x API is shaded into the
operator JAR.
+
+{{< hint warning >}}
+**Upgrading to Logback 1.4+/1.5+ or SLF4J 2.x is not supported.** SLF4J 2.x
uses a `ServiceLoader`-based binding mechanism that is incompatible with the
SLF4J 1.7.x API shaded inside the operator. Replacing the JARs at runtime will
result in `ClassNotFoundException: org.slf4j.impl.StaticLoggerBinder`.
+{{< /hint >}}
### FlinkDeployment Logging Configuration
-Users have the freedom to override the default `log4j-console.properties`
settings on a per-deployment level by putting the entire log configuration into
`spec.logConfiguration`:
+Users have the freedom to override the default logging settings on a
per-deployment level by putting the entire log configuration into
`spec.logConfiguration`.
+
+For Log4j2:
```yaml
spec:
@@ -656,3 +724,22 @@ spec:
rootLogger.appenderRef.file.ref = LogFile
...
```
+
+For Logback:
+
+```yaml
+spec:
+ ...
+ logConfiguration:
+ logback-console.xml: |+
+ <configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %-5level %-60.60logger{60} %X
- %msg%n</pattern>
+ </encoder>
+ </appender>
+ <root level="DEBUG">
+ <appender-ref ref="ConsoleAppender" />
+ </root>
+ </configuration>
+```
diff --git a/e2e-tests/test_logback_logging.sh
b/e2e-tests/test_logback_logging.sh
new file mode 100755
index 00000000..a557b26d
--- /dev/null
+++ b/e2e-tests/test_logback_logging.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# This script tests that the operator starts correctly with the logback logging
+# framework. The operator must be installed with logging.framework=logback
+# before running this test, e.g.:
+# helm install flink-operator helm/flink-kubernetes-operator --set
logging.framework=logback
+#
+# Verifications:
+# 1. Operator pod is running
+# 2. No SLF4J multiple-bindings warnings in logs
+# 3. Operator produces log output (proves logback is active)
+# 4. LOGGING_FRAMEWORK env var is set to "logback"
+# 5. LOG_CONFIG env var points to logback config
+# 6. logback-operator.xml is mounted in the container
+# 7. log4j-operator.properties is NOT mounted
+
+SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+source "${SCRIPT_DIR}/utils.sh"
+
+TIMEOUT=120
+
+passed=true
+
+operator_namespace=$(get_operator_pod_namespace)
+operator_pod=$(get_operator_pod_name)
+echo "Current operator pod is ${operator_pod} in namespace
${operator_namespace}"
+
+# Check that there are no SLF4J multiple-bindings warnings
+echo "Checking for SLF4J multiple bindings warnings..."
+if kubectl logs "${operator_pod}" -c flink-kubernetes-operator -n
"${operator_namespace}" | grep -q "SLF4J: Class path contains multiple SLF4J
bindings"; then
+ echo "ERROR: Found SLF4J multiple bindings warning"
+ passed=false
+fi
+
+# Verify logback is actually producing output
+echo "Checking that operator produces log output..."
+log_lines=$(kubectl logs "${operator_pod}" -c flink-kubernetes-operator -n
"${operator_namespace}" | wc -l)
+if [ "$log_lines" -lt 1 ]; then
+ echo "ERROR: No log output from operator"
+ passed=false
+fi
+
+# Verify log format
+echo "Verifying log format..."
+sample_log=$(kubectl logs "${operator_pod}" -c flink-kubernetes-operator -n
"${operator_namespace}" | grep "Starting Flink Kubernetes Operator" | head -1 |
sed 's/\x1b\[[0-9;]*m//g')
+
+# Check timestamp includes milliseconds (dot followed by 3 digits for logback)
+if ! echo "$sample_log" | grep -qE '^[0-9]{4}-[0-9]{2}-[0-9]{2}
[0-9]{2}:[0-9]{2}:[0-9]{2}[.][0-9]{3} '; then
+ echo "ERROR: Timestamp missing milliseconds. Sample: ${sample_log}"
+ passed=false
+fi
+
+if ! echo "$sample_log" | grep -qE 'o\.a\.f\.k\.o\.[A-Za-z]+'; then
+ echo "ERROR: Logger name not abbreviated. Sample: ${sample_log}"
+ passed=false
+fi
+
+if ! echo "$sample_log" | grep -qE '\[INFO \]'; then
+ echo "ERROR: Level format does not match [INFO ]. Sample: ${sample_log}"
+ passed=false
+fi
+
+# Verify LOGGING_FRAMEWORK env var
+echo "Verifying LOGGING_FRAMEWORK env var..."
+framework=$(kubectl get pod "${operator_pod}" -n "${operator_namespace}" -o
jsonpath='{.spec.containers[0].env[?(@.name=="LOGGING_FRAMEWORK")].value}')
+if [ "$framework" != "logback" ]; then
+ echo "ERROR: LOGGING_FRAMEWORK is '${framework}', expected 'logback'"
+ passed=false
+fi
+
+# Verify LOG_CONFIG env var
+echo "Verifying LOG_CONFIG env var..."
+log_config=$(kubectl get pod "${operator_pod}" -n "${operator_namespace}" -o
jsonpath='{.spec.containers[0].env[?(@.name=="LOG_CONFIG")].value}')
+if [[ "$log_config" != *"logback"* ]]; then
+ echo "ERROR: LOG_CONFIG is '${log_config}', expected it to contain 'logback'"
+ passed=false
+fi
+
+# Verify logback config file is mounted
+echo "Verifying logback-operator.xml is mounted..."
+if ! kubectl exec "${operator_pod}" -c flink-kubernetes-operator -n
"${operator_namespace}" -- test -f /opt/flink/conf/logback-operator.xml; then
+ echo "ERROR: logback-operator.xml not found in container"
+ passed=false
+fi
+
+# Verify log4j config files are NOT mounted
+echo "Verifying log4j-operator.properties is NOT mounted..."
+if kubectl exec "${operator_pod}" -c flink-kubernetes-operator -n
"${operator_namespace}" -- test -f /opt/flink/conf/log4j-operator.properties
2>/dev/null; then
+ echo "ERROR: log4j-operator.properties should not be mounted when using
logback"
+ passed=false
+fi
+
+if [ "$passed" = true ]; then
+ echo "Successfully run the logback logging framework test"
+else
+ echo "Logback logging framework test failed"
+ exit 1
+fi
+
diff --git a/flink-kubernetes-operator/pom.xml
b/flink-kubernetes-operator/pom.xml
index f4881275..6669472f 100644
--- a/flink-kubernetes-operator/pom.xml
+++ b/flink-kubernetes-operator/pom.xml
@@ -122,7 +122,9 @@ under the License.
<scope>provided</scope>
</dependency>
- <!-- Logging -->
+ <!-- Logging: SLF4J API and Log4j2 implementation are shaded into the
operator JAR.
+ Only the SLF4J bindings (log4j-slf4j-impl / logback-classic) are
excluded
+ and shipped as separate JARs so the entrypoint can pick one. -->
<dependency>
<groupId>org.slf4j</groupId>
@@ -134,6 +136,7 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>${log4j.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
@@ -154,6 +157,14 @@ under the License.
<version>${log4j.version}</version>
</dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+ <scope>provided</scope>
+ </dependency>
+
+
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@@ -311,6 +322,27 @@ under the License.
</executions>
<configuration>
<artifactItems>
+ <!-- Log4j2 SLF4J binding (default logging framework)
-->
+ <artifactItem>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j.version}</version>
+
<outputDirectory>${project.build.directory}/log4j</outputDirectory>
+ </artifactItem>
+ <!-- Logback SLF4J binding (alternative logging
framework) -->
+ <artifactItem>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>${logback.version}</version>
+
<outputDirectory>${project.build.directory}/logback</outputDirectory>
+ </artifactItem>
+ <artifactItem>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>${logback.version}</version>
+
<outputDirectory>${project.build.directory}/logback</outputDirectory>
+ </artifactItem>
+ <!-- Metrics plugins -->
<artifactItem>
<groupId>org.apache.flink</groupId>
<artifactId>flink-metrics-datadog</artifactId>
diff --git a/helm/flink-kubernetes-operator/conf/logback-console.xml
b/helm/flink-kubernetes-operator/conf/logback-console.xml
new file mode 100644
index 00000000..3e29f8a8
--- /dev/null
+++ b/helm/flink-kubernetes-operator/conf/logback-console.xml
@@ -0,0 +1,56 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %-5level %-60.60logger{60} %X
- %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="RollingFileAppender"
class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${log.file}</file>
+ <append>false</append>
+ <encoder>
+ <pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %-5level %-60.60logger{60} %X
- %msg%n</pattern>
+ </encoder>
+ <rollingPolicy
class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${log.file}.%i</fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>10</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>100MB</maxFileSize>
+ </triggeringPolicy>
+ </appender>
+
+ <!-- The following loggers keep the log level of common
libraries/connectors on INFO -->
+ <logger name="akka" level="INFO" />
+ <logger name="org.apache.kafka" level="INFO" />
+ <logger name="org.apache.hadoop" level="INFO" />
+ <logger name="org.apache.zookeeper" level="INFO" />
+
+ <!-- Suppress the irrelevant (wrong) warnings from the Netty channel
handler -->
+ <logger
name="org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline"
level="OFF" />
+
+ <root level="INFO">
+ <appender-ref ref="ConsoleAppender" />
+ <appender-ref ref="RollingFileAppender" />
+ </root>
+</configuration>
+
diff --git a/helm/flink-kubernetes-operator/conf/logback-operator.xml
b/helm/flink-kubernetes-operator/conf/logback-operator.xml
new file mode 100644
index 00000000..3e1e62b2
--- /dev/null
+++ b/helm/flink-kubernetes-operator/conf/logback-operator.xml
@@ -0,0 +1,35 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<configuration>
+ <appender name="ConsoleAppender"
class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%yellow(%d{yyyy-MM-dd HH:mm:ss.SSS})
%cyan(%-30logger{26})
%highlight([%-5level][%X{resource.namespace}/%X{resource.name}])
%msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <!-- Do not log config loading -->
+ <logger name="org.apache.flink.configuration.GlobalConfiguration"
level="ERROR" />
+
+ <!-- Avoid logging fallback key INFO messages -->
+ <logger name="org.apache.flink.configuration.Configuration" level="ERROR"
/>
+
+ <root level="INFO">
+ <appender-ref ref="ConsoleAppender" />
+ </root>
+</configuration>
diff --git a/helm/flink-kubernetes-operator/templates/_helpers.tpl
b/helm/flink-kubernetes-operator/templates/_helpers.tpl
index f4673d1e..e28eae05 100644
--- a/helm/flink-kubernetes-operator/templates/_helpers.tpl
+++ b/helm/flink-kubernetes-operator/templates/_helpers.tpl
@@ -77,3 +77,15 @@ Create the path of the operator image to use
{{- .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag
}}
{{- end }}
{{- end }}
+
+{{/*
+Determine the JVM log configuration flag based on the selected logging
framework.
+Supported values for .Values.logging.framework: "log4j2" (default), "logback".
+*/}}
+{{- define "flink-operator.logConfig" -}}
+{{- if eq .Values.logging.framework "logback" -}}
+-Dlogback.configurationFile=/opt/flink/conf/logback-operator.xml
+{{- else -}}
+-Dlog4j.configurationFile=/opt/flink/conf/log4j-operator.properties
+{{- end -}}
+{{- end }}
diff --git a/helm/flink-kubernetes-operator/templates/controller/configmap.yaml
b/helm/flink-kubernetes-operator/templates/controller/configmap.yaml
index d8695481..5ab29cd8 100644
--- a/helm/flink-kubernetes-operator/templates/controller/configmap.yaml
+++ b/helm/flink-kubernetes-operator/templates/controller/configmap.yaml
@@ -64,5 +64,17 @@ data:
{{- end }}
{{- if index (.Values.defaultConfiguration) "log4j-console.properties" }}
{{- index (.Values.defaultConfiguration) "log4j-console.properties" |
nindent 4 -}}
+{{- end }}
+ logback-operator.xml: |+
+{{- if index (.Values.defaultConfiguration) "logback-operator.xml" }}
+ {{- index (.Values.defaultConfiguration) "logback-operator.xml" | nindent 4
-}}
+{{- else if .Values.defaultConfiguration.append }}
+ {{- $.Files.Get "conf/logback-operator.xml" | nindent 4 -}}
+{{- end }}
+ logback-console.xml: |+
+{{- if index (.Values.defaultConfiguration) "logback-console.xml" }}
+ {{- index (.Values.defaultConfiguration) "logback-console.xml" | nindent 4
-}}
+{{- else if .Values.defaultConfiguration.append }}
+ {{- $.Files.Get "conf/logback-console.xml" | nindent 4 -}}
{{- end }}
{{- end }}
diff --git
a/helm/flink-kubernetes-operator/templates/controller/deployment.yaml
b/helm/flink-kubernetes-operator/templates/controller/deployment.yaml
index 85e6a465..a31175c9 100644
--- a/helm/flink-kubernetes-operator/templates/controller/deployment.yaml
+++ b/helm/flink-kubernetes-operator/templates/controller/deployment.yaml
@@ -116,8 +116,10 @@ spec:
value: /opt/flink/conf
- name: FLINK_PLUGINS_DIR
value: /opt/flink/plugins
+ - name: LOGGING_FRAMEWORK
+ value: {{ .Values.logging.framework }}
- name: LOG_CONFIG
- value: {{ .Values.jvmArgs.logConfig }}
+ value: {{ include "flink-operator.logConfig" . }}
- name: JVM_ARGS
value: {{ .Values.jvmArgs.operator }}
{{- if .Values.tls.create }}
@@ -198,13 +200,15 @@ spec:
- name: WEBHOOK_SERVER_PORT
value: "9443"
- name: LOG_CONFIG
- value: {{ .Values.jvmArgs.logConfig }}
+ value: {{ include "flink-operator.logConfig" . }}
- name: JVM_ARGS
value: {{ .Values.jvmArgs.webhook }}
- name: FLINK_CONF_DIR
value: /opt/flink/conf
- name: FLINK_PLUGINS_DIR
value: /opt/flink/plugins
+ - name: LOGGING_FRAMEWORK
+ value: {{ .Values.logging.framework }}
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
@@ -247,10 +251,17 @@ spec:
- key: config.yaml
path: config.yaml
{{- end }}
+ {{- if eq .Values.logging.framework "logback" }}
+ - key: logback-operator.xml
+ path: logback-operator.xml
+ - key: logback-console.xml
+ path: logback-console.xml
+ {{- else }}
- key: log4j-operator.properties
path: log4j-operator.properties
- key: log4j-console.properties
path: log4j-console.properties
+ {{- end }}
{{- if .Values.operatorVolumes.create }}
{{- toYaml .Values.operatorVolumes.data | nindent 8 }}
{{- else }}
diff --git
a/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml
b/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml
index b9dc1277..90476695 100644
--- a/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml
+++ b/helm/flink-kubernetes-operator/tests/controller/configmap_test.yaml
@@ -36,3 +36,35 @@ tests:
kind: ConfigMap
name: flink-operator-config
namespace: flink-operator
+
+ - it: Should include logback-operator.xml key in configmap
+ set:
+ defaultConfiguration:
+ create: true
+ asserts:
+ - isNotNull:
+ path: data["logback-operator.xml"]
+
+ - it: Should include logback-console.xml key in configmap
+ set:
+ defaultConfiguration:
+ create: true
+ asserts:
+ - isNotNull:
+ path: data["logback-console.xml"]
+
+ - it: Should include log4j-operator.properties key in configmap
+ set:
+ defaultConfiguration:
+ create: true
+ asserts:
+ - isNotNull:
+ path: data["log4j-operator.properties"]
+
+ - it: Should include log4j-console.properties key in configmap
+ set:
+ defaultConfiguration:
+ create: true
+ asserts:
+ - isNotNull:
+ path: data["log4j-console.properties"]
diff --git
a/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml
b/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml
index 8769721e..e6333d76 100644
--- a/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml
+++ b/helm/flink-kubernetes-operator/tests/controller/deployment_test.yaml
@@ -303,3 +303,83 @@ tests:
- equal:
path: spec.template.spec.serviceAccountName
value: test-service-account
+
+# Logging framework tests
+
+- it: Should use log4j2 logging by default
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: LOGGING_FRAMEWORK
+ value: log4j2
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: LOG_CONFIG
+ value:
-Dlog4j.configurationFile=/opt/flink/conf/log4j-operator.properties
+
+- it: Should mount log4j config files by default
+ asserts:
+ - contains:
+ path: spec.template.spec.volumes[0].configMap.items
+ content:
+ key: log4j-operator.properties
+ path: log4j-operator.properties
+ - contains:
+ path: spec.template.spec.volumes[0].configMap.items
+ content:
+ key: log4j-console.properties
+ path: log4j-console.properties
+
+- it: Should use logback logging when `logging.framework` is set to logback
+ set:
+ logging:
+ framework: logback
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: LOGGING_FRAMEWORK
+ value: logback
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: LOG_CONFIG
+ value: -Dlogback.configurationFile=/opt/flink/conf/logback-operator.xml
+
+- it: Should mount logback config files when `logging.framework` is set to
logback
+ set:
+ logging:
+ framework: logback
+ asserts:
+ - contains:
+ path: spec.template.spec.volumes[0].configMap.items
+ content:
+ key: logback-operator.xml
+ path: logback-operator.xml
+ - contains:
+ path: spec.template.spec.volumes[0].configMap.items
+ content:
+ key: logback-console.xml
+ path: logback-console.xml
+
+- it: Should not mount logback config files when using log4j2
+ asserts:
+ - notContains:
+ path: spec.template.spec.volumes[0].configMap.items
+ content:
+ key: logback-operator.xml
+ path: logback-operator.xml
+
+- it: Should not mount log4j config files when using logback
+ set:
+ logging:
+ framework: logback
+ asserts:
+ - notContains:
+ path: spec.template.spec.volumes[0].configMap.items
+ content:
+ key: log4j-operator.properties
+ path: log4j-operator.properties
+
diff --git a/helm/flink-kubernetes-operator/values.yaml
b/helm/flink-kubernetes-operator/values.yaml
index 497bcc71..c9242f2b 100644
--- a/helm/flink-kubernetes-operator/values.yaml
+++ b/helm/flink-kubernetes-operator/values.yaml
@@ -194,6 +194,13 @@ defaultConfiguration:
log4j-console.properties: |+
# Flink Deployment Logging Overrides
# rootLogger.level = DEBUG
+ logback-operator.xml:
+ logback-console.xml:
+
+# Logging framework to use for the operator. Supported values: "log4j2",
"logback".
+# This controls which config file is passed to the JVM and which files are
mounted.
+logging:
+ framework: "log4j2"
# (Optional) Exposes metrics port on the container if defined
metrics:
@@ -206,7 +213,6 @@ fullnameOverride: ""
jvmArgs:
webhook: ""
operator: ""
- logConfig:
"-Dlog4j.configurationFile=/opt/flink/conf/log4j-operator.properties"
# Configure health probes for the operator
operatorHealth:
diff --git a/pom.xml b/pom.xml
index f3a0a8bc..693124a2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@ under the License.
<slf4j.version>1.7.36</slf4j.version>
<log4j.version>2.23.1</log4j.version>
+ <logback.version>1.2.13</logback.version>
<spotless.version>2.40.0</spotless.version>
<it.skip>true</it.skip>
