This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new f5922bc2730 [fix][broker] Sanitize values before logging in
apply-config-from-env.py script (#22044)
f5922bc2730 is described below
commit f5922bc2730bac2c455bb9866f1ba581897abd4f
Author: Lari Hotari <lhot...@users.noreply.github.com>
AuthorDate: Thu Feb 8 21:43:26 2024 -0800
[fix][broker] Sanitize values before logging in apply-config-from-env.py
script (#22044)
(cherry picked from commit 303678364eab538c16041214cae1588a5b2111d9)
---
.../scripts/apply-config-from-env-with-prefix.py | 99 ++++------------------
docker/pulsar/scripts/apply-config-from-env.py | 57 ++++++-------
2 files changed, 45 insertions(+), 111 deletions(-)
diff --git a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py
b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py
index 3f6bc2e4d3b..9943b283a9f 100755
--- a/docker/pulsar/scripts/apply-config-from-env-with-prefix.py
+++ b/docker/pulsar/scripts/apply-config-from-env-with-prefix.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,85 +18,22 @@
# under the License.
#
-##
-## Edit a properties config file and replace values based on
-## the ENV variables
-## export prefix_my-key=new-value
-## ./apply-config-from-env-with-prefix prefix_ file.conf
-##
-
-import os, sys
-
-if len(sys.argv) < 3:
- print('Usage: %s' % (sys.argv[0]))
- sys.exit(1)
-
-# Always apply env config to env scripts as well
-prefix = sys.argv[1]
-conf_files = sys.argv[2:]
-
-PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1')
-
-for conf_filename in conf_files:
- lines = [] # List of config file lines
- keys = {} # Map a key to its line number in the file
-
- # Load conf file
- for line in open(conf_filename):
- lines.append(line)
- line = line.strip()
- if not line or line.startswith('#'):
- continue
-
- try:
- k,v = line.split('=', 1)
- keys[k] = len(lines) - 1
- except:
- if PF_ENV_DEBUG:
- print("[%s] skip Processing %s" % (conf_filename, line))
-
- # Update values from Env
- for k in sorted(os.environ.keys()):
- v = os.environ[k].strip()
-
- # Hide the value in logs if is password.
- if "password" in k.lower():
- displayValue = "********"
- else:
- displayValue = v
-
- if k.startswith(prefix):
- k = k[len(prefix):]
- if k in keys:
- print('[%s] Applying config %s = %s' % (conf_filename, k,
displayValue))
- idx = keys[k]
- lines[idx] = '%s=%s\n' % (k, v)
-
-
- # Add new keys from Env
- for k in sorted(os.environ.keys()):
- v = os.environ[k]
- if not k.startswith(prefix):
- continue
-
- # Hide the value in logs if is password.
- if "password" in k.lower():
- displayValue = "********"
- else:
- displayValue = v
-
- k = k[len(prefix):]
- if k not in keys:
- print('[%s] Adding config %s = %s' % (conf_filename, k,
displayValue))
- lines.append('%s=%s\n' % (k, v))
- else:
- print('[%s] Updating config %s = %s' % (conf_filename, k,
displayValue))
- lines[keys[k]] = '%s=%s\n' % (k, v)
-
+############################################################
+# Edit a properties config file and replace values based on
+# the ENV variables
+# export prefix_my-key=new-value
+# ./apply-config-from-env-with-prefix prefix_ file.conf
+#
+# Environment variables that are prefixed with the command
+# line prefix will be used to updated file properties if
+# they exist and create new ones if they don't.
+#
+# Environment variables not prefixed will be used only to
+# update if they exist and ignored if they don't.
+############################################################
- # Store back the updated config in the same file
- f = open(conf_filename, 'w')
- for line in lines:
- f.write(line)
- f.close()
+# DEPRECATED: Use "apply-config-from-env.py --prefix MY_PREFIX_ conf_file"
instead
+# this is not a python script, but a bash script. Call
apply-config-from-env.py with the prefix argument
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+"${SCRIPT_DIR}/apply-config-from-env.py" --prefix "$1" "${@:2}"
diff --git a/docker/pulsar/scripts/apply-config-from-env.py
b/docker/pulsar/scripts/apply-config-from-env.py
index a802ca6fdbe..269d3fd221f 100755
--- a/docker/pulsar/scripts/apply-config-from-env.py
+++ b/docker/pulsar/scripts/apply-config-from-env.py
@@ -25,18 +25,29 @@
## ./apply-config-from-env file.conf
##
-import os, sys
-
-if len(sys.argv) < 2:
- print('Usage: %s' % (sys.argv[0]))
+import os, sys, argparse
+
+parser = argparse.ArgumentParser(description='Pulsar configuration file
customizer based on environment variables')
+parser.add_argument('--prefix', default='PULSAR_PREFIX_', help='Prefix for
environment variables, default is PULSAR_PREFIX_')
+parser.add_argument('conf_files', nargs='*', help='Configuration files')
+args = parser.parse_args()
+if not args.conf_files:
+ parser.print_help()
sys.exit(1)
-# Always apply env config to env scripts as well
-conf_files = sys.argv[1:]
+env_prefix = args.prefix
+conf_files = args.conf_files
-PF_ENV_PREFIX = 'PULSAR_PREFIX_'
PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1')
+# List of keys where the value should not be displayed in logs
+sensitive_keys = ["brokerClientAuthenticationParameters",
"bookkeeperClientAuthenticationParameters", "tokenSecretKey"]
+
+def sanitize_display_value(k, v):
+ if "password" in k.lower() or k in sensitive_keys or (k ==
"tokenSecretKey" and v.startswith("data:")):
+ return "********"
+ return v
+
for conf_filename in conf_files:
lines = [] # List of config file lines
keys = {} # Map a key to its line number in the file
@@ -47,7 +58,6 @@ for conf_filename in conf_files:
line = line.strip()
if not line:
continue
-
try:
k,v = line.split('=', 1)
if k.startswith('#'):
@@ -61,37 +71,26 @@ for conf_filename in conf_files:
for k in sorted(os.environ.keys()):
v = os.environ[k].strip()
- # Hide the value in logs if is password.
- if "password" in k.lower():
- displayValue = "********"
- else:
- displayValue = v
-
- if k.startswith(PF_ENV_PREFIX):
- k = k[len(PF_ENV_PREFIX):]
if k in keys:
+ displayValue = sanitize_display_value(k, v)
print('[%s] Applying config %s = %s' % (conf_filename, k,
displayValue))
idx = keys[k]
lines[idx] = '%s=%s\n' % (k, v)
-
# Ensure we have a new-line at the end of the file, to avoid issue
# when appending more lines to the config
lines.append('\n')
-
- # Add new keys from Env
+
+ # Add new keys from Env
for k in sorted(os.environ.keys()):
- v = os.environ[k]
- if not k.startswith(PF_ENV_PREFIX):
+ if not k.startswith(env_prefix):
continue
- # Hide the value in logs if is password.
- if "password" in k.lower():
- displayValue = "********"
- else:
- displayValue = v
+ v = os.environ[k].strip()
+ k = k[len(env_prefix):]
+
+ displayValue = sanitize_display_value(k, v)
- k = k[len(PF_ENV_PREFIX):]
if k not in keys:
print('[%s] Adding config %s = %s' % (conf_filename, k,
displayValue))
lines.append('%s=%s\n' % (k, v))
@@ -99,10 +98,8 @@ for conf_filename in conf_files:
print('[%s] Updating config %s = %s' % (conf_filename, k,
displayValue))
lines[keys[k]] = '%s=%s\n' % (k, v)
-
# Store back the updated config in the same file
f = open(conf_filename, 'w')
for line in lines:
f.write(line)
- f.close()
-
+ f.close()
\ No newline at end of file
