AlinsRan commented on code in PR #13347: URL: https://github.com/apache/apisix/pull/13347#discussion_r3230863918
########## docs/zh/latest/plugins/data-mask.md: ########## @@ -0,0 +1,297 @@ +--- +title: data-mask +keywords: + - APISIX + - API 网关 + - Plugin + - data-mask +description: API 网关 Apache APISIX data-mask 插件可用于在请求数据写入访问日志或日志插件之前，对敏感字段进行掩码或脱敏处理。 +--- + +<!-- +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +--> + +<head> + <link rel="canonical" href="https://docs.api7.ai/hub/data-mask"; /> +</head> + +## 描述 + +`data-mask` 插件可在请求数据（查询参数、请求头、请求体）写入访问日志或日志插件（如 `file-logger`、`http-logger`）之前，对敏感字段进行掩码或脱敏处理。 + +该插件适用于防止凭证、令牌、支付卡号及其他敏感信息被写入日志的场景。 + +插件在 `log` 阶段运行，支持以下三种掩码动作： + +- `remove`：从请求数据中完全删除该字段。 +- `replace`：将字段值替换为固定字符串。 +- `regex`：对字段值执行正则表达式替换。 + Review Comment: Fixed in 66e2cab8: 在描述章节中补充了关于 `$request_line` 的说明及配置示例。 ########## t/plugin/data-mask.t: ########## @@ -0,0 +1,724 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (! $block->request) { + $block->set_value("request", "GET /t"); + if (!$block->response_body) { + $block->set_value("response_body", "passed\n"); + } + } +}); + + +run_tests; + +__DATA__ + +=== TEST 1: mask query +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.1" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 2: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello?password=abc&token=xyz&card=1234-1234-1234-1234", ngx.HTTP_GET) + local fd, err = io.open("mask-query.log.1", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.querystring.password then + ngx.say("password arg mask failed: " .. log.request.querystring.password) + return + end + if log.request.querystring.token ~= "*****" then + ngx.say("token arg mask failed: " .. log.request.querystring.token) + return + end + if log.request.querystring.card ~= "1234-****-****-1234" then + ngx.say("card arg mask failed: " .. log.request.querystring.card) + return + end + if log.request.uri ~= "/hello?token=*****&card=1234-****-****-1234" and + log.request.uri ~= "/hello?card=1234-****-****-1234&token=*****" then + ngx.say("uri mask failed: " .. log.request.uri) + return + end + + os.remove("mask-query.log.1") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 3: mask header +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "header" + }, + { + "action": "replace", + "name": "token", + "type": "header", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "header", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-header.log.2" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 4: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local headers = {} + headers["password"] = "abc" + headers["token"] = "xyz" + headers["card"] = "1234-1234-1234-1234" + local code = t("/hello", ngx.HTTP_GET, "", nil, headers) + + local fd, err = io.open("mask-header.log.2", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.headers.password then + ngx.say("password header mask failed: " .. log.request.headers.password) + return + end + if log.request.headers.token ~= "*****" then + ngx.say("token header mask failed: " .. log.request.headers.token) + return + end + if log.request.headers.card ~= "1234-****-****-1234" then + ngx.say("card header mask failed: " .. log.request.headers.card) + return + end + + os.remove("mask-header.log.2") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 5: mask urlencoded body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "urlencoded", + "name": "password", + "type": "body" + }, + { + "action": "replace", + "body_format": "urlencoded", + "name": "token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "urlencoded", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 6: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + local fd, err = io.open("mask-urlencoded-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.body ~= "token=*****&card=1234-****-****-1234" and + log.request.body ~= "card=1234-****-****-1234&token=*****" then + ngx.say("urlencoded body mask failed: " .. log.request.body) + return + end + + os.remove("mask-urlencoded-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 7: mask json body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "json", + "name": "$.password", + "type": "body" + }, + { + "action": "replace", + "body_format": "json", + "name": "users[*].token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "json", + "name": "$.users[*].credit.card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-json-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 8: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", + ngx.HTTP_POST, + [[{ + "password": "abc", + "users": [ + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + }, + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + } + ] + }]] + ) + + local fd, err = io.open("mask-json-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + local body = core.json.decode(log.request.body) + if body.password then + ngx.say("$.password mask failed: " .. body.password) + return + end + for _, user in ipairs(body.users) do + if user.token ~= "*****" then + ngx.say("$.users[*].token mask failed: " .. user.token) + return + end + if user.credit.card ~= "1234-****-****-1234" then + ngx.say("$.users[*].credit.card mask failed: " .. user.credit.card) + return + end + end + + os.remove("mask-json-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 9: plugin within global rule should not throw error for missing body. +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.4" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 10: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/random", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + ngx.say("code: ", code) + } + } +--- response_body +code: 404 +--- no_error_log +no request body found + + + +=== TEST 11: create plugin with default value for `max_req_post_args` +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "regex", + "body_format": "urlencoded", + "name": "arg100", + "regex": "(\\d+)$", + "type": "body", + "value": "$1" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 12: verify default value for `max_req_post_args`` +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local url_encoded = "arg1=1" + for i = 2, 110, 1 do + url_encoded = url_encoded .. "&arg" .. i .. "=" .. i + end + + local code = t("/hello", ngx.HTTP_POST, url_encoded) + + local fd, err = io.open("mask-urlencoded-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + local match100, err = ngx.re.match(log.request.body, "arg100=100") + local match101, err = ngx.re.match(log.request.body, "arg101=101") + os.remove("mask-urlencoded-body.log") + if match100 and not match101 then + ngx.say("success") + return + end + ngx.say(match) Review Comment: Fixed in 66e2cab8: Replaced the undefined `match` variable with `"failed: match100=" .. tostring(match100) .. ", match101=" .. tostring(match101)` for meaningful debug output. Same fix applied on line 655. ########## t/plugin/data-mask.t: ########## @@ -0,0 +1,724 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +no_long_string(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (! $block->request) { + $block->set_value("request", "GET /t"); + if (!$block->response_body) { + $block->set_value("response_body", "passed\n"); + } + } +}); + + +run_tests; + +__DATA__ + +=== TEST 1: mask query +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.1" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 2: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello?password=abc&token=xyz&card=1234-1234-1234-1234", ngx.HTTP_GET) + local fd, err = io.open("mask-query.log.1", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.querystring.password then + ngx.say("password arg mask failed: " .. log.request.querystring.password) + return + end + if log.request.querystring.token ~= "*****" then + ngx.say("token arg mask failed: " .. log.request.querystring.token) + return + end + if log.request.querystring.card ~= "1234-****-****-1234" then + ngx.say("card arg mask failed: " .. log.request.querystring.card) + return + end + if log.request.uri ~= "/hello?token=*****&card=1234-****-****-1234" and + log.request.uri ~= "/hello?card=1234-****-****-1234&token=*****" then + ngx.say("uri mask failed: " .. log.request.uri) + return + end + + os.remove("mask-query.log.1") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 3: mask header +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "header" + }, + { + "action": "replace", + "name": "token", + "type": "header", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "header", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-header.log.2" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 4: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local headers = {} + headers["password"] = "abc" + headers["token"] = "xyz" + headers["card"] = "1234-1234-1234-1234" + local code = t("/hello", ngx.HTTP_GET, "", nil, headers) + + local fd, err = io.open("mask-header.log.2", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.headers.password then + ngx.say("password header mask failed: " .. log.request.headers.password) + return + end + if log.request.headers.token ~= "*****" then + ngx.say("token header mask failed: " .. log.request.headers.token) + return + end + if log.request.headers.card ~= "1234-****-****-1234" then + ngx.say("card header mask failed: " .. log.request.headers.card) + return + end + + os.remove("mask-header.log.2") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 5: mask urlencoded body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "urlencoded", + "name": "password", + "type": "body" + }, + { + "action": "replace", + "body_format": "urlencoded", + "name": "token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "urlencoded", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 6: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + local fd, err = io.open("mask-urlencoded-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + if log.request.body ~= "token=*****&card=1234-****-****-1234" and + log.request.body ~= "card=1234-****-****-1234&token=*****" then + ngx.say("urlencoded body mask failed: " .. log.request.body) + return + end + + os.remove("mask-urlencoded-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 7: mask json body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "body_format": "json", + "name": "$.password", + "type": "body" + }, + { + "action": "replace", + "body_format": "json", + "name": "users[*].token", + "type": "body", + "value": "*****" + }, + { + "action": "regex", + "body_format": "json", + "name": "$.users[*].credit.card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "body", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-json-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 8: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/hello", + ngx.HTTP_POST, + [[{ + "password": "abc", + "users": [ + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + }, + { + "token": "xyz", + "credit": { + "card": "1234-1234-1234-1234" + } + } + ] + }]] + ) + + local fd, err = io.open("mask-json-body.log", "r") + if not fd then + core.log.error("failed to open file: ", err) + return + end + local line = fd:read() + local log = core.json.decode(line) + + local body = core.json.decode(log.request.body) + if body.password then + ngx.say("$.password mask failed: " .. body.password) + return + end + for _, user in ipairs(body.users) do + if user.token ~= "*****" then + ngx.say("$.users[*].token mask failed: " .. user.token) + return + end + if user.credit.card ~= "1234-****-****-1234" then + ngx.say("$.users[*].credit.card mask failed: " .. user.credit.card) + return + end + end + + os.remove("mask-json-body.log") + ngx.say("success") + } + } +--- response_body +success + + + +=== TEST 9: plugin within global rule should not throw error for missing body. +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "remove", + "name": "password", + "type": "query" + }, + { + "action": "replace", + "name": "token", + "type": "query", + "value": "*****" + }, + { + "action": "regex", + "name": "card", + "regex": "(\\d+)\\-\\d+\\-\\d+\\-(\\d+)", + "type": "query", + "value": "$1-****-****-$2" + } + ] + }, + "file-logger": { + "path": "mask-query.log.4" + } + } + }]] + ) + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 10: verify +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local code = t("/random", ngx.HTTP_POST, "password=abc&token=xyz&card=1234-1234-1234-1234") + + ngx.say("code: ", code) + } + } +--- response_body +code: 404 +--- no_error_log +no request body found + + + +=== TEST 11: create plugin with default value for `max_req_post_args` +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "data-mask": { + "request": [ + { + "action": "regex", + "body_format": "urlencoded", + "name": "arg100", + "regex": "(\\d+)$", + "type": "body", + "value": "$1" + } + ] + }, + "file-logger": { + "include_req_body": true, + "path": "mask-urlencoded-body.log" + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } + + + +=== TEST 12: verify default value for `max_req_post_args`` Review Comment: Fixed in 66e2cab8: Removed the extra trailing backtick from the TEST 12 title. ########## apisix/plugins/data-mask.lua: ########## @@ -0,0 +1,265 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local ngx = ngx +local ipairs = ipairs +local next = next +local type = type +local re_sub = ngx.re.sub +local core = require("apisix.core") +local jp = require("jsonpath") + +local plugin_name = "data-mask" + +local schema = { + type = "object", + properties = { + request = { + type = "array", + items = { + type = "object", + properties = { + type = {type = "string", enum = {"query", "header", "body"}}, + body_format = {type = "string", enum = {"json", "urlencoded"}}, + name = {type = "string"}, + action = {type = "string", enum = {"regex", "replace", "remove"}}, + regex = {type = "string"}, + value = {type = "string"}, + }, + required = {"type", "name", "action"}, + allOf = { + { + ["if"] = { + properties = {type = {const = "body"}}, + }, + ["then"] = { + required = {"body_format"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "regex"}}, + }, + ["then"] = { + required = {"regex", "value"}, + }, + }, + { + ["if"] = { + properties = {action = {const = "replace"}}, + }, + ["then"] = { + required = {"value"}, + }, + }, + }, + }, + }, + max_body_size = { + type = "integer", + exclusiveMinimum = 0, + default = 1024 * 1024, + }, + max_req_post_args = { + type = "integer", + default = 100, + minimum = 0, + } + }, +} + + +local _M = { + version = 0.1, + priority = 1500, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err + end + return true +end + + +local function regex_replace(origin, regex, new) + local res, _, err = re_sub(origin, regex, new, "jo") + if not res then + core.log.error("failed to replace (" .. origin .. ") by regex (".. regex .. + ") with new value (" .. new .. "): ", err) + end + return res +end + + +local function mask_table(tab, conf) + if not tab[conf.name] then + return false + end + local masked = false + if conf.action == "remove" then + tab[conf.name] = nil + masked = true + elseif conf.action == "replace" then + tab[conf.name] = conf.value + masked = true + elseif conf.action == "regex" then + local new_arg = regex_replace(tab[conf.name], conf.regex, conf.value) + if new_arg then + tab[conf.name] = new_arg + masked = true + end + end + return masked +end + + +-- jsonpath index of array starts from 0, lua table index starts from 1 +local function table_index(idx) + if type(idx) == "number" then + return idx + 1 + end + return idx +end + + +local function mask_json(obj, conf) + -- local nodes = jp.nodes(data, '$..author') + -- { + -- { path = {'$', 'store', 'book', 0, 'author'}, value = 'Nigel Rees' }, + -- { path = {'$', 'store', 'book', 1, 'author'}, value = 'Evelyn Waugh' }, + -- } + local nodes = jp.nodes(obj, conf.name) + if not nodes then + return false + end + + local masked = false + for _, node in ipairs(nodes) do + local nested = obj + -- first element is root($), last element is the field name + for i = 2, #node.path - 1 do + nested = nested[table_index(node.path[i])] + end + local index = table_index(node.path[#node.path]) + if conf.action == "remove" then + nested[index] = nil + elseif conf.action == "replace" then + nested[index] = conf.value + elseif conf.action == "regex" then + nested[index] = regex_replace(node.value, conf.regex, conf.value) + end + masked = true + end + return masked +end + + +function _M.log(conf, ctx) + local args = core.request.get_uri_args(ctx) + local query_masked = false + local post_args = {} + local post_args_masked = false + local body = ngx.req.get_body_data() + if body then + post_args = ngx.req.get_post_args(conf.max_req_post_args) + end + local json_body + local body_masked = false + + if conf.request then + for _, item in ipairs(conf.request) do + if item.type == "query" then + if mask_table(args, item) then + query_masked = true + end + end + + if item.type == "header" then + local header = core.request.header(ctx, item.name) + if header then + if item.action == "remove" then + core.request.set_header(ctx, item.name, nil) + elseif item.action == "replace" then + core.request.set_header(ctx, item.name, item.value) + elseif item.action == "regex" then + core.request.set_header(ctx, item.name, + regex_replace(header, item.regex, item.value)) + end + end + end + + if item.type == "body" then + if item.body_format == "urlencoded" then + if mask_table(post_args, item) then + post_args_masked = true + end + elseif item.body_format == "json" then Review Comment: Fixed in 66e2cab8: `get_post_args()` is now deferred — it is only called when the first `urlencoded` body rule is encountered during the rules loop. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
