AlinsRan commented on PR #13686:
URL: https://github.com/apache/apisix/pull/13686#issuecomment-5114732172
Concrete sketch of the token approach, in case it helps the discussion. The
exported API changes from `resolve_conf_var_in_text(text) -> text` to
`load_yaml_conf(text) -> conf, err`, because resolution now happens after
parsing; the two call sites are `apisix/cli/file.lua:321` and
`apisix/core/config_yaml.lua:101`.
```lua
local TOKEN_PREFIX = "__APISIX_ENV_VAR_"
local TOKEN_SUFFIX = "__"
-- pass 1: replace every `${{VAR}}` reference with a token that survives YAML
-- parsing as a plain scalar. References inside comments become tokens too,
but
-- the YAML parser drops them, so an unset var in a comment never surfaces.
local function tokenize_conf_var(text)
local refs = {}
local parts = {}
local pos = 1
while true do
local from, to, body = str_find(text, "%$%{%{%s*(.-)%s*%}%}", pos)
if not from then
break
end
local var, default = body, nil
local i = str_find(body, "%:%=")
if i then
var = str_sub(body, 1, i - 1)
default = str_sub(body, i + 2):gsub("^%s*(.-)%s*$", "%1")
end
-- a reference wrapped in quotes stays a string, an unquoted one
takes
-- its type from the value (#13078)
local quote = str_sub(text, from - 1, from - 1)
local quoted = (quote == '"' or quote == "'")
and str_sub(text, to + 1, to + 1) == quote
refs[#refs + 1] = {var = var, default = default, quoted = quoted}
parts[#parts + 1] = str_sub(text, pos, from - 1)
parts[#parts + 1] = TOKEN_PREFIX .. #refs .. TOKEN_SUFFIX
pos = to + 1
end
parts[#parts + 1] = str_sub(text, pos)
return table_concat(parts), refs
end
local function lookup(ref)
local v = getenv(ref.var) or ref.default
if not v then
return nil, "failed to handle configuration: " ..
"can't find environment variable " .. ref.var
end
if not exported_vars then
exported_vars = {}
end
exported_vars[ref.var] = v -- still feeds get_exported_vars()
return v
end
-- pass 2: resolve the tokens that made it into the parsed config
local function resolve_token(str, refs)
local idx = str:match("^" .. TOKEN_PREFIX .. "(%d+)" .. TOKEN_SUFFIX ..
"$")
if idx then
local ref = refs[tonumber(idx)]
local v, err = lookup(ref)
if not v then
return nil, err
end
if ref.quoted then
return v
end
if tonumber(v) then
return tonumber(v)
elseif v == "true" then
return true
elseif v == "false" then
return false
end
return v
end
local err
local new_str = str:gsub(TOKEN_PREFIX .. "(%d+)" .. TOKEN_SUFFIX,
function(i)
local v, e = lookup(refs[tonumber(i)])
if not v then
err = e
return ""
end
return v
end)
if err then
return nil, err
end
return new_str
end
local function resolve_conf_var_in_tree(conf, refs)
local renamed_keys
for key, val in pairs(conf) do
if type(val) == "table" then
local ok, err = resolve_conf_var_in_tree(val, refs)
if not ok then
return nil, err
end
elseif type(val) == "string" then
local new_val, err = resolve_token(val, refs)
if err then
return nil, err
end
conf[key] = new_val
end
if type(key) == "string" then
local new_key, err = resolve_token(key, refs)
if err then
return nil, err
end
if new_key ~= key then
renamed_keys = renamed_keys or {}
renamed_keys[#renamed_keys + 1] = {key, tostring(new_key)}
end
end
end
-- deferred, inserting a key while iterating with pairs() is undefined
for _, pair in ipairs(renamed_keys or {}) do
conf[pair[2]] = conf[pair[1]]
conf[pair[1]] = nil
end
return true
end
function _M.load_yaml_conf(text)
local tokenized, refs = tokenize_conf_var(text)
local ok, conf = pcall(yaml.load, tokenized)
if not ok then
return nil, "invalid yaml: " .. tostring(conf)
end
if type(conf) ~= "table" then
return conf
end
local resolved, err = resolve_conf_var_in_tree(conf, refs)
if not resolved then
return nil, err
end
return conf
end
```
Output of that sketch on the cases from my previous comment (`W=3 B=true
K=mykey HASHY='secret # not-a-comment' MULTI=$'a\nb: 2'
BIG=356002209726529540`):
```
full-line comment {"key":1}
inline comment {"key":1}
missing var ERR: failed to handle configuration: can't find
environment variable UNSET_XYZ
unquoted number {"weight":3}
quoted number {"key":"3"}
unquoted bool {"flag":true}
default fallback {"key":"fallback"}
var in key {"mykey":1}
embedded string {"key":"pre-3-post"}
value with # {"key":"secret # not-a-comment"}
multiline value {"key":"a\nb: 2"}
block scalar {"body":"# 3 stays\n"}
big int quoted {"id":"356002209726529540"}
two refs one line {"key":"mykey-3"}
exported_vars
{"B":"true","BIG":"356002209726529540","HASHY":"secret # not-a-comment",
"K":"mykey","MULTI":"a\nb:
2","NOPE":"fallback","W":"3"}
```
Two notes on it:
- `exported_vars` is still populated during pass 2, so `get_exported_vars()`
and the generated nginx `env` directives keep working — the resolution just
moves later.
- token collision is the one thing to guard: if the raw text already
contains `TOKEN_PREFIX`, bail out or pick another prefix.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
