[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r762665326
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,256 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local ngx = ngx
+local timer_at = ngx.timer.at
+local buffers = {}
+
+local lrucache = core.lrucache.new({
+type = "plugin",
+})
+
+local schema = {
+type = "object",
+properties = {
+meta_format = {
+type = "string",
+default = "default",
+enum = {"default", "origin"},
+},
+nameserver_list = {
+type = "array",
+minItems = 1,
+items = {
+type = "string"
+}
+},
+topic = {type = "string"},
+key = {type = "string"},
+tag = {type = "string"},
+timeout = {type = "integer", minimum = 1, default = 3},
+use_tls = {type = "boolean", default = false},
+access_key = {type = "string", default = ""},
+secret_key = {type = "string", default = ""},
+name = {type = "string", default = "rocketmq logger"},
+max_retry_count = {type = "integer", minimum = 0, default = 0},
+retry_delay = {type = "integer", minimum = 0, default = 1},
+buffer_duration = {type = "integer", minimum = 1, default = 60},
+inactive_timeout = {type = "integer", minimum = 1, default = 5},
+include_req_body = {type = "boolean", default = false},
+include_req_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+include_resp_body = {type = "boolean", default = false},
+include_resp_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+},
+required = {"nameserver_list", "topic"}
+}
+
+local metadata_schema = {
+type = "object",
+properties = {
+log_format = log_util.metadata_schema_log_format,
+},
+}
+
+local _M = {
+version = 0.1,
+priority = 402,
+name = plugin_name,
+schema = schema,
+metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+if schema_type == core.schema.TYPE_METADATA then
+return core.schema.check(metadata_schema, conf)
+end
+
+local ok, err = core.schema.check(schema, conf)
+if not ok then
+return nil, err
+end
+return log_util.check_log_schema(conf)
+end
+
+
+-- remove stale objects from the memory after timer expires
+local function remove_stale_objects(premature)
+if premature then
+return
+end
+
+for key, batch in ipairs(buffers) do
+if #batch.entry_buffer.entries == 0 and #batch.batch_to_process == 0
then
+core.log.warn("removing batch processor stale object, conf: ",
+ core.json.delay_encode(key))
+buffers[key] = nil
+end
+end
+
+stale_timer_running = false
+end
+
+
+local function create_producer(nameserver_list, producer_config)
+core.log.info("create new rocketmq producer instance")
+local prod = producer.new(nameserver_list, "apisixLogProducer")
+if producer_config.use_tls then
+prod:setUseTLS(true)
+end
+if producer_config.access_key ~= '' then
+local aclHook = acl_rpchook.new(producer_config.access_key,
producer_config.secret_key)
+prod:addRPCHook(aclHook)
+end
+prod:setTimeout(producer_config.timeout
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r762658128
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,256 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local ngx = ngx
+local timer_at = ngx.timer.at
+local buffers = {}
+
+local lrucache = core.lrucache.new({
+type = "plugin",
+})
+
+local schema = {
+type = "object",
+properties = {
+meta_format = {
+type = "string",
+default = "default",
+enum = {"default", "origin"},
+},
+nameserver_list = {
+type = "array",
+minItems = 1,
+items = {
+type = "string"
+}
+},
+topic = {type = "string"},
+key = {type = "string"},
+tag = {type = "string"},
+timeout = {type = "integer", minimum = 1, default = 3},
+use_tls = {type = "boolean", default = false},
+access_key = {type = "string", default = ""},
+secret_key = {type = "string", default = ""},
+name = {type = "string", default = "rocketmq logger"},
+max_retry_count = {type = "integer", minimum = 0, default = 0},
+retry_delay = {type = "integer", minimum = 0, default = 1},
+buffer_duration = {type = "integer", minimum = 1, default = 60},
+inactive_timeout = {type = "integer", minimum = 1, default = 5},
+include_req_body = {type = "boolean", default = false},
+include_req_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+include_resp_body = {type = "boolean", default = false},
+include_resp_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+},
+required = {"nameserver_list", "topic"}
+}
+
+local metadata_schema = {
+type = "object",
+properties = {
+log_format = log_util.metadata_schema_log_format,
+},
+}
+
+local _M = {
+version = 0.1,
+priority = 402,
+name = plugin_name,
+schema = schema,
+metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+if schema_type == core.schema.TYPE_METADATA then
+return core.schema.check(metadata_schema, conf)
+end
+
+local ok, err = core.schema.check(schema, conf)
+if not ok then
+return nil, err
+end
+return log_util.check_log_schema(conf)
+end
+
+
+-- remove stale objects from the memory after timer expires
+local function remove_stale_objects(premature)
+if premature then
+return
+end
+
+for key, batch in ipairs(buffers) do
+if #batch.entry_buffer.entries == 0 and #batch.batch_to_process == 0
then
+core.log.warn("removing batch processor stale object, conf: ",
+ core.json.delay_encode(key))
+buffers[key] = nil
+end
+end
+
+stale_timer_running = false
+end
+
+
+local function create_producer(nameserver_list, producer_config)
+core.log.info("create new rocketmq producer instance")
+local prod = producer.new(nameserver_list, "apisixLogProducer")
+if producer_config.use_tls then
+prod:setUseTLS(true)
+end
+if producer_config.access_key ~= '' then
+local aclHook = acl_rpchook.new(producer_config.access_key,
producer_config.secret_key)
+prod:addRPCHook(aclHook)
+end
+prod:setTimeout(producer_config.timeout
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r762658128
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,256 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local ngx = ngx
+local timer_at = ngx.timer.at
+local buffers = {}
+
+local lrucache = core.lrucache.new({
+type = "plugin",
+})
+
+local schema = {
+type = "object",
+properties = {
+meta_format = {
+type = "string",
+default = "default",
+enum = {"default", "origin"},
+},
+nameserver_list = {
+type = "array",
+minItems = 1,
+items = {
+type = "string"
+}
+},
+topic = {type = "string"},
+key = {type = "string"},
+tag = {type = "string"},
+timeout = {type = "integer", minimum = 1, default = 3},
+use_tls = {type = "boolean", default = false},
+access_key = {type = "string", default = ""},
+secret_key = {type = "string", default = ""},
+name = {type = "string", default = "rocketmq logger"},
+max_retry_count = {type = "integer", minimum = 0, default = 0},
+retry_delay = {type = "integer", minimum = 0, default = 1},
+buffer_duration = {type = "integer", minimum = 1, default = 60},
+inactive_timeout = {type = "integer", minimum = 1, default = 5},
+include_req_body = {type = "boolean", default = false},
+include_req_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+include_resp_body = {type = "boolean", default = false},
+include_resp_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+},
+required = {"nameserver_list", "topic"}
+}
+
+local metadata_schema = {
+type = "object",
+properties = {
+log_format = log_util.metadata_schema_log_format,
+},
+}
+
+local _M = {
+version = 0.1,
+priority = 402,
+name = plugin_name,
+schema = schema,
+metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+if schema_type == core.schema.TYPE_METADATA then
+return core.schema.check(metadata_schema, conf)
+end
+
+local ok, err = core.schema.check(schema, conf)
+if not ok then
+return nil, err
+end
+return log_util.check_log_schema(conf)
+end
+
+
+-- remove stale objects from the memory after timer expires
+local function remove_stale_objects(premature)
+if premature then
+return
+end
+
+for key, batch in ipairs(buffers) do
+if #batch.entry_buffer.entries == 0 and #batch.batch_to_process == 0
then
+core.log.warn("removing batch processor stale object, conf: ",
+ core.json.delay_encode(key))
+buffers[key] = nil
+end
+end
+
+stale_timer_running = false
+end
+
+
+local function create_producer(nameserver_list, producer_config)
+core.log.info("create new rocketmq producer instance")
+local prod = producer.new(nameserver_list, "apisixLogProducer")
+if producer_config.use_tls then
+prod:setUseTLS(true)
+end
+if producer_config.access_key ~= '' then
+local aclHook = acl_rpchook.new(producer_config.access_key,
producer_config.secret_key)
+prod:addRPCHook(aclHook)
+end
+prod:setTimeout(producer_config.timeout
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r762658128
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,256 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local ngx = ngx
+local timer_at = ngx.timer.at
+local buffers = {}
+
+local lrucache = core.lrucache.new({
+type = "plugin",
+})
+
+local schema = {
+type = "object",
+properties = {
+meta_format = {
+type = "string",
+default = "default",
+enum = {"default", "origin"},
+},
+nameserver_list = {
+type = "array",
+minItems = 1,
+items = {
+type = "string"
+}
+},
+topic = {type = "string"},
+key = {type = "string"},
+tag = {type = "string"},
+timeout = {type = "integer", minimum = 1, default = 3},
+use_tls = {type = "boolean", default = false},
+access_key = {type = "string", default = ""},
+secret_key = {type = "string", default = ""},
+name = {type = "string", default = "rocketmq logger"},
+max_retry_count = {type = "integer", minimum = 0, default = 0},
+retry_delay = {type = "integer", minimum = 0, default = 1},
+buffer_duration = {type = "integer", minimum = 1, default = 60},
+inactive_timeout = {type = "integer", minimum = 1, default = 5},
+include_req_body = {type = "boolean", default = false},
+include_req_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+include_resp_body = {type = "boolean", default = false},
+include_resp_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+},
+required = {"nameserver_list", "topic"}
+}
+
+local metadata_schema = {
+type = "object",
+properties = {
+log_format = log_util.metadata_schema_log_format,
+},
+}
+
+local _M = {
+version = 0.1,
+priority = 402,
+name = plugin_name,
+schema = schema,
+metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+if schema_type == core.schema.TYPE_METADATA then
+return core.schema.check(metadata_schema, conf)
+end
+
+local ok, err = core.schema.check(schema, conf)
+if not ok then
+return nil, err
+end
+return log_util.check_log_schema(conf)
+end
+
+
+-- remove stale objects from the memory after timer expires
+local function remove_stale_objects(premature)
+if premature then
+return
+end
+
+for key, batch in ipairs(buffers) do
+if #batch.entry_buffer.entries == 0 and #batch.batch_to_process == 0
then
+core.log.warn("removing batch processor stale object, conf: ",
+ core.json.delay_encode(key))
+buffers[key] = nil
+end
+end
+
+stale_timer_running = false
+end
+
+
+local function create_producer(nameserver_list, producer_config)
+core.log.info("create new rocketmq producer instance")
+local prod = producer.new(nameserver_list, "apisixLogProducer")
+if producer_config.use_tls then
+prod:setUseTLS(true)
+end
+if producer_config.access_key ~= '' then
+local aclHook = acl_rpchook.new(producer_config.access_key,
producer_config.secret_key)
+prod:addRPCHook(aclHook)
+end
+prod:setTimeout(producer_config.timeout
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r762656485
##
File path: t/plugin/rocketmq-logger.t
##
@@ -0,0 +1,1098 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_root_location();
+run_tests;
+
+__DATA__
+
+=== TEST 1: sanity
+--- config
+location /t {
+content_by_lua_block {
+local plugin = require("apisix.plugins.rocketmq-logger")
+local ok, err = plugin.check_schema({
+ topic = "test",
+ key = "key1",
+ nameserver_list = {
+"127.0.0.1:3"
+ }
+})
+if not ok then
+ngx.say(err)
+end
+ngx.say("done")
+}
+}
+--- request
+GET /t
+--- response_body
+done
+--- no_error_log
+[error]
+
+
+
+=== TEST 2: missing nameserver list
+--- config
+location /t {
+content_by_lua_block {
+local plugin = require("apisix.plugins.rocketmq-logger")
+local ok, err = plugin.check_schema({topic = "test", key= "key1"})
+if not ok then
+ngx.say(err)
+end
+ngx.say("done")
+}
+}
+--- request
+GET /t
+--- response_body
+property "nameserver_list" is required
+done
+--- no_error_log
+[error]
+
+
+
+=== TEST 3: wrong type of string
+--- config
+location /t {
+content_by_lua_block {
+local plugin = require("apisix.plugins.rocketmq-logger")
+local ok, err = plugin.check_schema({
+nameserver_list = {
+"127.0.0.1:3000"
+},
+timeout = "10",
+topic ="test",
+key= "key1"
+})
+if not ok then
+ngx.say(err)
+end
+ngx.say("done")
+}
+}
+--- request
+GET /t
+--- response_body
+property "timeout" validation failed: wrong type: expected integer, got string
+done
+--- no_error_log
+[error]
+
+
+
+=== TEST 4: set route(id: 1)
+--- config
+location /t {
+content_by_lua_block {
+local t = require("lib.test_admin").test
+local code, body = t('/apisix/admin/routes/1',
+ ngx.HTTP_PUT,
+ [[{
+"plugins": {
+"rocketmq-logger": {
+"nameserver_list" : [ "127.0.0.1:9876" ],
+"topic" : "test2",
+"key" : "key1",
+"timeout" : 1,
+"batch_max_size": 1
+}
+},
+"upstream": {
+"nodes": {
+"127.0.0.1:1980": 1
+},
+"type": "roundrobin"
+},
+"uri": "/hello"
+}]],
+[[{
+"node": {
+"value": {
+"plugins": {
+ "rocketmq-logger": {
+"nameserver_list" : [ "127.0.0.1:9876" ],
+"topic" : "test2",
+"key" : "key1",
+"timeout" : 1,
+"batch_max_size": 1
+}
+},
+"upstream": {
+"nodes": {
+"127.0.0.1:1980": 1
+},
+"type": "roundrobin"
+},
+"uri": "/hello"
+},
+"key": "/apisix/routes/1"
+},
+"action": "set"
+}]]
+)
+if code >= 300 then
+ngx.status = code
+
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r762654690
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,256 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local ngx = ngx
+local timer_at = ngx.timer.at
+local buffers = {}
+
+local lrucache = core.lrucache.new({
+type = "plugin",
+})
+
+local schema = {
+type = "object",
+properties = {
+meta_format = {
+type = "string",
+default = "default",
+enum = {"default", "origin"},
+},
+nameserver_list = {
+type = "array",
+minItems = 1,
+items = {
+type = "string"
+}
+},
+topic = {type = "string"},
+key = {type = "string"},
+tag = {type = "string"},
+timeout = {type = "integer", minimum = 1, default = 3},
+use_tls = {type = "boolean", default = false},
+access_key = {type = "string", default = ""},
+secret_key = {type = "string", default = ""},
+name = {type = "string", default = "rocketmq logger"},
+max_retry_count = {type = "integer", minimum = 0, default = 0},
+retry_delay = {type = "integer", minimum = 0, default = 1},
+buffer_duration = {type = "integer", minimum = 1, default = 60},
+inactive_timeout = {type = "integer", minimum = 1, default = 5},
+include_req_body = {type = "boolean", default = false},
+include_req_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+include_resp_body = {type = "boolean", default = false},
+include_resp_body_expr = {
+type = "array",
+minItems = 1,
+items = {
+type = "array",
+items = {
+type = "string"
+}
+}
+},
+},
+required = {"nameserver_list", "topic"}
+}
+
+local metadata_schema = {
+type = "object",
+properties = {
+log_format = log_util.metadata_schema_log_format,
+},
+}
+
+local _M = {
+version = 0.1,
+priority = 402,
+name = plugin_name,
+schema = schema,
+metadata_schema = metadata_schema,
+}
+
+
+function _M.check_schema(conf, schema_type)
+if schema_type == core.schema.TYPE_METADATA then
+return core.schema.check(metadata_schema, conf)
+end
+
+local ok, err = core.schema.check(schema, conf)
+if not ok then
+return nil, err
+end
+return log_util.check_log_schema(conf)
+end
+
+
+-- remove stale objects from the memory after timer expires
+local function remove_stale_objects(premature)
+if premature then
+return
+end
+
+for key, batch in ipairs(buffers) do
Review comment:
fixed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653: URL: https://github.com/apache/apisix/pull/5653#discussion_r760838391 ## File path: docs/zh/latest/plugins/rocketmq-logger.md ## @@ -0,0 +1,229 @@ +--- +title: rocketmq-logger +--- + + + +## 目录 + +- [**简介**](#简介) +- [**属性**](#属性) +- [**工作原理**](#工作原理) +- [**如何启用**](#如何启用) +- [**测试插件**](#测试插件) +- [**禁用插件**](#禁用插件) + +## 简介 + +`rocketmq-logger` 插件利用ngx_lua客户端能力，可推送JSON格式的请求日志到外部rocketmq集群。 + +它可以将接口请求日志以 JSON 的形式推送给外部 rocketmq 集群。如果在短时间内没有收到日志数据，请放心，它会在我们的批处理处理器中的计时器功能到期后自动发送日志。 Review comment: ok -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653: URL: https://github.com/apache/apisix/pull/5653#discussion_r760838167 ## File path: docs/en/latest/plugins/rocketmq-logger.md ## @@ -0,0 +1,234 @@ +--- +title: rocketmq-logger +--- + + + +## Summary + +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**Info**](#info) +- [**How To Enable**](#how-to-enable) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + +## Name + +`rocketmq-logger` is a plugin which provides the ability to push requests log data as JSON objects to your external rocketmq clusters. + +This plugin provides the ability to push requests log data as JSON objects to your external rocketmq clusters. In case if you did not receive the log data don't worry give it some time it will automatically send the logs after the timer function expires in our Batch Processor. Review comment: duplication deleted now , that was copied from kafka-logger -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653: URL: https://github.com/apache/apisix/pull/5653#discussion_r760837218 ## File path: docs/en/latest/plugins/rocketmq-logger.md ## @@ -0,0 +1,234 @@ +--- +title: rocketmq-logger +--- + + + +## Summary + +- [**Name**](#name) +- [**Attributes**](#attributes) +- [**Info**](#info) +- [**How To Enable**](#how-to-enable) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + +## Name + +`rocketmq-logger` is a plugin which provides the ability to push requests log data as JSON objects to your external rocketmq clusters. + +This plugin provides the ability to push requests log data as JSON objects to your external rocketmq clusters. In case if you did not receive the log data don't worry give it some time it will automatically send the logs after the timer function expires in our Batch Processor. + +For more info on Batch-Processor in Apache APISIX please refer. +[Batch-Processor](../batch-processor.md) + +## Attributes + +| Name | Type| Requirement | Default| Valid | Description | +| | --- | --- | -- | --- | | +| nameserver_list | object | required|| | An array of rocketmq nameservers. | +| topic| string | required|| | Target topic to push data. | +| key | string | optional|| | Keys of messages to send. | +| tag | string | optional || | Tags of messages to send. | +| timeout | integer | optional| 3 | [1,...] | Timeout for the upstream to send data. | +| use_tls | boolean | optional | false | | Whether to open TLS | +| access_key | string | optional | "" | | access key for ACL, empty string means disable ACL. | +| secret_key | string | optional | "" | | secret key for ACL。 | +| name | string | optional| "rocketmq logger" | | A unique identifier to identity the batch processor. | +| meta_format | enum| optional| "default" | ["default"，"origin"] | `default`: collect the request information with default JSON way. `origin`: collect the request information with original HTTP request. [example](#examples-of-meta_format)| +| batch_max_size | integer | optional| 1000 | [1,...] | Set the maximum number of logs sent in each batch. When the number of logs reaches the set maximum, all logs will be automatically pushed to the `rocketmq` service. | +| inactive_timeout | integer | optional| 5 | [1,...] | The maximum time to refresh the buffer (in seconds). When the maximum refresh time is reached, all logs will be automatically pushed to the `rocketmq` service regardless of whether the number of logs in the buffer reaches the set maximum number. | +| buffer_duration | integer | optional| 60 | [1,...] | Maximum age in seconds of the oldest entry in a batch before the batch must be processed.| +| max_retry_count | integer | optional| 0 | [0,...] | Maximum number of retries before removing from the processing pipe line. | +| retry_delay | integer | optional| 1 | [0,...] | Number of seconds the process execution should be delayed if the execution fails. | +| include_req_body | boolean | optional| false | [false, true] | Whether to include the request body. false: indicates that the requested body is not included; true: indicates that the requested body is included. Note: if the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitation. | +| include_req_body_expr | array | optional| | | When `include_req_body` is true, control the behavior based on the result of the [lua-resty-expr](https://github.com/api7/lua-resty-expr) expression. If present, only log the request body when the result is true. | +| include_resp_body| boolean | optional| false | [false, true] | Whether to include the response body. The response body is included if and only if it is `true`. | +| include_resp_body_expr | array | optional| | | When `include_resp_body` is true, control the behavior based on the result of the [l
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r760837115
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,271 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local pairs= pairs
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local timer_at = ngx.timer.at
+local ngx = ngx
+local buffers = {}
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653: URL: https://github.com/apache/apisix/pull/5653#discussion_r760836989 ## File path: ci/pod/docker-compose.yml ## @@ -355,10 +355,34 @@ services: networks: apisix_net: + namesrv: Review comment: @leslie-tsang thanks, I found another way -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r760727687
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,271 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local pairs= pairs
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local timer_at = ngx.timer.at
+local ngx = ngx
+local buffers = {}
+
+local lrucache = core.lrucache.new({
+type = "plugin",
+})
+
+local schema = {
+type = "object",
+properties = {
+meta_format = {
+type = "string",
+default = "default",
+enum = {"default", "origin"},
+},
+nameserver_list = {
+type = "object",
+minProperties = 1,
+patternProperties = {
Review comment:
chaged to [ "127.0.0.1:9876" ]
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653:
URL: https://github.com/apache/apisix/pull/5653#discussion_r760727805
##
File path: apisix/plugins/rocketmq-logger.lua
##
@@ -0,0 +1,271 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local log_util = require("apisix.utils.log-util")
+local producer = require ("resty.rocketmq.producer")
+local acl_rpchook = require("resty.rocketmq.acl_rpchook")
+local batch_processor = require("apisix.utils.batch-processor")
+local plugin = require("apisix.plugin")
+
+local pairs= pairs
+local type = type
+local ipairs = ipairs
+local plugin_name = "rocketmq-logger"
+local stale_timer_running = false
+local timer_at = ngx.timer.at
+local ngx = ngx
+local buffers = {}
+
+local lrucache = core.lrucache.new({
+type = "plugin",
+})
+
+local schema = {
+type = "object",
+properties = {
+meta_format = {
+type = "string",
+default = "default",
+enum = {"default", "origin"},
+},
+nameserver_list = {
+type = "object",
+minProperties = 1,
+patternProperties = {
+[".*"] = {
+description = "the port of rocketmq nameserver",
+type = "integer",
+minimum = 1,
+maximum = 65535,
+},
+},
+},
+rocketmq_topic = {type = "string"},
Review comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653: URL: https://github.com/apache/apisix/pull/5653#discussion_r760723620 ## File path: ci/pod/docker-compose.yml ## @@ -355,10 +355,34 @@ services: networks: apisix_net: + namesrv: Review comment: the conf in contaner use this name as a host address, if changed, broker will fail to connect nameserver. and I cant find a way to change the config file, do you have any idea? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [apisix] yuz10 commented on a change in pull request #5653: feat: rocketmq logger
yuz10 commented on a change in pull request #5653: URL: https://github.com/apache/apisix/pull/5653#discussion_r760089341 ## File path: docs/zh/latest/plugins/rocketmq-logger.md ## @@ -0,0 +1,232 @@ +--- +title: rocketmq-logger +--- + + + +## 目录 + +- [**简介**](#简介) +- [**属性**](#属性) +- [**工作原理**](#工作原理) +- [**如何启用**](#如何启用) +- [**测试插件**](#测试插件) +- [**禁用插件**](#禁用插件) + +## 简介 + +`rocketmq-logger` 是一个插件，可用作ngx_lua nginx 模块的 rocketmq 客户端驱动程序。 Review comment: fixed. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: notifications-unsubscr...@apisix.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
