AlinsRan commented on code in PR #13347: URL: https://github.com/apache/apisix/pull/13347#discussion_r3230863408
########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res Review Comment: Fixed in 66e2cab8: `regex_replace()` now only logs the regex error message without including the original value or replacement string. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked Review Comment: Fixed in 66e2cab8: `mask_table()` now checks if the value is a table (multi-value param) and applies the regex to each entry individually. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) + end + masked = true + end + return masked +end + + +function _M.log(conf, ctx) + local args = core.request.get_uri_args(ctx) + local query_masked = false + local post_args = {} + local post_args_masked = false + local body = ngx.req.get_body_data() + if body then + post_args = ngx.req.get_post_args(conf.max_req_post_args) + end + local json_body + local body_masked = false + + if conf.request then + for _, item in ipairs(conf.request) do + if item.type == "query" then + if mask_table(args, item) then + query_masked = true + end + end + + if item.type == "header" then + local header = core.request.header(ctx, item.name) + if header then + if item.action == "remove" then + core.request.set_header(ctx, item.name, nil) + elseif item.action == "replace" then + core.request.set_header(ctx, item.name, item.value) + elseif item.action == "regex" then + core.request.set_header(ctx, item.name, + regex_replace(header, item.regex, item.value)) + end + end + end + + if item.type == "body" then + if item.body_format == "urlencoded" then + if mask_table(post_args, item) then + post_args_masked = true + end + elseif item.body_format == "json" then + if body and #body <= conf.max_body_size then + if not json_body then + local js, err = core.json.decode(body) + if not js then + core.log.warn("failed to decode json body: ", err) + else + json_body = js + end + end + if json_body then + if mask_json(json_body, item) then + body_masked = true + end + end + elseif body and #body > conf.max_body_size then + core.log.warn("data-mask: skipping body masking for field '", + item.name, "' because body size (", #body, + ") exceeds max_body_size (", conf.max_body_size, ")") + end + + end + end Review Comment: Fixed in 66e2cab8: Added a warn log when `ngx.req.get_body_file()` returns a path (body stored in temp file). Full reading of body files in the log phase is out of scope for this plugin — `max_body_size` is the intended mechanism to control memory usage, and bodies buffered to disk would typically exceed that limit. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) Review Comment: Fixed in 66e2cab8: `mask_json()` now only updates `nested[index]` when `regex_replace()` returns a non-nil result, and the `masked` flag is set per-action rather than unconditionally. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) + end + masked = true Review Comment: Fixed in 66e2cab8: Added a type check before calling `regex_replace()` — non-string values emit a warn log and are left unchanged. ########## docs/en/latest/plugins/data-mask.md: ########## @@ -0,0 +1,296 @@ +--- +title: data-mask +keywords: + - Apache APISIX + - API Gateway + - Plugin + - data-mask +description: This document contains information about the Apache APISIX data-mask Plugin. +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +<head> + <link rel="canonical" href="https://docs.api7.ai/hub/data-mask"; /> +</head> + +## Description + +The `data-mask` Plugin masks or redacts sensitive fields in request data — query parameters, headers, and body — before they appear in access logs or logger plugins (such as `file-logger` or `http-logger`). + +This is useful for preventing credentials, tokens, payment card numbers, and other sensitive information from being written to logs. + +The plugin runs in the `log` phase and supports three masking actions: + +- `remove`: completely removes the field from the request data. +- `replace`: replaces the field value with a fixed string. +- `regex`: applies a regular expression substitution to the field value. Review Comment: Fixed in 66e2cab8: Added a note in the Description section documenting the `$request_line` access_log_format requirement with a config example. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
