This is an automated email from the ASF dual-hosted git repository. spacewander pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push: new 139c397 fix(log-rotate): after enabling compression collect log exceptions (#5715) 139c397 is described below commit 139c3972898f1e45c6d5b1f40143c504116b3486 Author: 帅进超 <shuaijinc...@gmail.com> AuthorDate: Wed Dec 8 09:28:03 2021 +0800 fix(log-rotate): after enabling compression collect log exceptions (#5715) --- apisix/plugins/log-rotate.lua | 202 ++++++++++++++++++++++++++++-------------- t/plugin/log-rotate.t | 59 ++++++++++-- t/plugin/log-rotate2.t | 54 +++++++++-- 3 files changed, 233 insertions(+), 82 deletions(-) diff --git a/apisix/plugins/log-rotate.lua b/apisix/plugins/log-rotate.lua index 273dc01..5b711c9 100644 --- a/apisix/plugins/log-rotate.lua +++ b/apisix/plugins/log-rotate.lua @@ -22,12 +22,21 @@ local process = require("ngx.process") local signal = require("resty.signal") local shell = require("resty.shell") local ngx = ngx +local ngx_time = ngx.time +local ngx_update_time = ngx.update_time local lfs = require("lfs") -local io = io -local os = os -local table = table -local string = string -local str_find = core.string.find +local type = type +local io_open = io.open +local os_date = os.date +local os_remove = os.remove +local os_rename = os.rename +local str_sub = string.sub +local str_find = string.find +local str_format = string.format +local str_reverse = string.reverse +local tab_insert = table.insert +local tab_sort = table.sort + local local_conf @@ -35,7 +44,11 @@ local plugin_name = "log-rotate" local INTERVAL = 60 * 60 -- rotate interval (unit: second) local MAX_KEPT = 24 * 7 -- max number of log files will be kept local COMPRESSION_FILE_SUFFIX = ".tar.gz" -- compression file suffix +local rotate_time +local default_logs local enable_compression = false +local DEFAULT_ACCESS_LOG_FILENAME = "access.log" +local DEFAULT_ERROR_LOG_FILENAME = "error.log" local schema = { type = "object", @@ -53,7 +66,7 @@ local _M = { local function file_exists(path) - local file = io.open(path, "r") + local file = io_open(path, "r") if file then file:close() end @@ -62,7 +75,7 @@ end local function get_last_index(str, key) - local rev = string.reverse(str) + local rev = str_reverse(str) local _, idx = str_find(rev, key) local n if idx then @@ -88,15 +101,15 @@ local function get_log_path_info(file_type) local prefix = ngx.config.prefix() if conf_path then - local root = string.sub(conf_path, 1, 1) + local root = str_sub(conf_path, 1, 1) -- relative path if root ~= "/" then conf_path = prefix .. conf_path end local n = get_last_index(conf_path, "/") if n ~= nil and n ~= #conf_path then - local dir = string.sub(conf_path, 1, n) - local name = string.sub(conf_path, n + 1) + local dir = str_sub(conf_path, 1, n) + local name = str_sub(conf_path, n + 1) return dir, name end end @@ -105,50 +118,11 @@ local function get_log_path_info(file_type) end -local function rotate_file(date_str, file_type) - local log_dir, filename = get_log_path_info(file_type) - - core.log.info("rotate log_dir:", log_dir) - core.log.info("rotate filename:", filename) - - local new_filename = date_str .. "__" .. filename - local file_path = log_dir .. new_filename - if file_exists(file_path) then - core.log.info("file exist: ", file_path) - return false - end - - local file_path_org = log_dir .. filename - local ok, msg = os.rename(file_path_org, file_path) - core.log.info("move file from ", file_path_org, " to ", file_path, - " res:", ok, " msg:", msg) - - if ok and enable_compression then - local compression_filename = new_filename .. COMPRESSION_FILE_SUFFIX - local cmd = string.format("cd %s && tar -zcf %s %s", - log_dir, compression_filename, new_filename) - core.log.info("log file compress command: " .. cmd) - local ok, stdout, stderr, reason, status = shell.run(cmd) - core.log.info("compress log file from ", new_filename, " to ", compression_filename, - " res:", ok) - - if ok then - ok = os.remove(file_path) - core.log.warn("remove uncompressed log file: ", file_path, " ret: ", ok) - else - core.log.error("failed to compress log file: ", new_filename, " ret: ", ok, - " stdout: ", stdout, " stderr: ", stderr, " reason: ", reason, " status: ", status) - end - end - - return true -end - - -local function tab_sort(a, b) +local function tab_sort_comp(a, b) return a > b end + local function scan_log_folder() local t = { access = {}, @@ -168,19 +142,83 @@ local function scan_log_folder() if n ~= nil then local log_type = file:sub(n + 2) if log_type == access_name then - table.insert(t.access, file) + tab_insert(t.access, file) elseif log_type == error_name then - table.insert(t.error, file) + tab_insert(t.error, file) end end end - table.sort(t.access, tab_sort) - table.sort(t.error, tab_sort) + tab_sort(t.access, tab_sort_comp) + tab_sort(t.error, tab_sort_comp) return t, log_dir end +local function rename_file(log, date_str) + local new_file + if not log.new_file then + core.log.warn(log.type, " is off") + return + end + + new_file = str_format(log.new_file, date_str) + if file_exists(new_file) then + core.log.info("file exist: ", new_file) + return new_file + end + + local ok, err = os_rename(log.file, new_file) + if not ok then + core.log.error("move file from ", log.file, " to ", new_file, + " res:", ok, " msg:", err) + return + end + + return new_file +end + + +local function compression_file(new_file) + if not new_file or type(new_file) ~= "string" then + core.log.info("compression file: ", new_file, " invalid") + return + end + + local n = get_last_index(new_file, "/") + local new_filepath = str_sub(new_file, 1, n) + local new_filename = str_sub(new_file, n + 1) + local com_filename = new_filename .. COMPRESSION_FILE_SUFFIX + local cmd = str_format("cd %s && tar -zcf %s %s", new_filepath, + com_filename, new_filename) + core.log.info("log file compress command: " .. cmd) + + local ok, stdout, stderr, reason, status = shell.run(cmd) + if not ok then + core.log.error("compress log file from ", new_filename, " to ", com_filename, + " fail, stdout: ", stdout, " stderr: ", stderr, " reason: ", reason, + " status: ", status) + return + end + + ok, stderr = os_remove(new_file) + if stderr then + core.log.error("remove uncompressed log file: ", new_file, + " fail, err: ", stderr, " res:", ok) + end +end + + +local function init_default_logs(logs_info, log_type) + local filepath, filename = get_log_path_info(log_type) + logs_info[log_type] = { type = log_type } + if filename ~= "off" then + logs_info[log_type].file = filepath .. filename + logs_info[log_type].new_file = filepath .. "/%s__" .. filename + end +end + + local function rotate() local interval = INTERVAL local max_kept = MAX_KEPT @@ -194,18 +232,34 @@ local function rotate() core.log.info("rotate interval:", interval) core.log.info("rotate max keep:", max_kept) - local time = ngx.time() - if time % interval == 0 then - time = time - interval - else - time = time - time % interval + if not default_logs then + -- first init default log filepath and filename + default_logs = {} + init_default_logs(default_logs, DEFAULT_ACCESS_LOG_FILENAME) + init_default_logs(default_logs, DEFAULT_ERROR_LOG_FILENAME) end - local date_str = os.date("%Y-%m-%d_%H-%M-%S", time) + ngx_update_time() + local now_time = ngx_time() + if not rotate_time then + -- first init rotate time + rotate_time = now_time + interval + core.log.info("first init rotate time is: ", rotate_time) + return + end + + if now_time < rotate_time then + -- did not reach the rotate time + core.log.info("rotate time: ", rotate_time, " now time: ", now_time) + return + end - local ok1 = rotate_file(date_str, "access.log") - local ok2 = rotate_file(date_str, "error.log") - if not ok1 and not ok2 then + local now_date = os_date("%Y-%m-%d_%H-%M-%S", now_time) + local access_new_file = rename_file(default_logs[DEFAULT_ACCESS_LOG_FILENAME], now_date) + local error_new_file = rename_file(default_logs[DEFAULT_ERROR_LOG_FILENAME], now_date) + if not access_new_file and not error_new_file then + -- reset rotate time + rotate_time = rotate_time + interval return end @@ -216,19 +270,31 @@ local function rotate() core.log.error("failed to send USR1 signal for reopening log file: ", err) end + if enable_compression then + compression_file(access_new_file) + compression_file(error_new_file) + end + -- clean the oldest file local log_list, log_dir = scan_log_folder() for i = max_kept + 1, #log_list.error do local path = log_dir .. log_list.error[i] - local ok = os.remove(path) - core.log.warn("remove old error file: ", path, " ret: ", ok) + ok, err = os_remove(path) + if err then + core.log.error("remove old error file: ", path, " err: ", err, " res:", ok) + end end for i = max_kept + 1, #log_list.access do local path = log_dir .. log_list.access[i] - local ok = os.remove(path) - core.log.warn("remove old access file: ", path, " ret: ", ok) + ok, err = os_remove(path) + if err then + core.log.error("remove old error file: ", path, " err: ", err, " res:", ok) + end end + + -- reset rotate time + rotate_time = rotate_time + interval end diff --git a/t/plugin/log-rotate.t b/t/plugin/log-rotate.t index 94f5ac9..5e04be1 100644 --- a/t/plugin/log-rotate.t +++ b/t/plugin/log-rotate.t @@ -40,7 +40,16 @@ plugin_attr: _EOC_ $block->set_value("yaml_config", $user_yaml_config); - $block->set_value("request", "GET /t"); + + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + }); run_tests; @@ -81,8 +90,6 @@ __DATA__ } --- error_code eval [200] ---- no_error_log -[error] @@ -97,8 +104,6 @@ __DATA__ } --- response_body done ---- no_error_log -[error] --- error_log start xxxxxx @@ -139,8 +144,6 @@ start xxxxxx } --- response_body done ---- no_error_log -[error] @@ -181,5 +184,43 @@ plugins: --- response_body done true ---- no_error_log -[error] + + + +=== TEST 5: check file changes (disable compression) +--- config + location /t { + content_by_lua_block { + ngx.sleep(2) + + local default_logs = {} + for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do + if string.match(file_name, "__error.log$") or string.match(file_name, "__access.log$") then + local filepath = ngx.config.prefix() .. "/logs/" .. file_name + local attr = lfs.attributes(filepath) + if attr then + default_logs[filepath] = { change = attr.change, size = attr.size } + end + end + end + + ngx.sleep(1) + + local passed = false + for filepath, origin_attr in pairs(default_logs) do + local check_attr = lfs.attributes(filepath) + if check_attr.change == origin_attr.change and check_attr.size == origin_attr.size then + passed = true + else + passed = false + break + end + end + + if passed then + ngx.say("passed") + end + } + } +--- response_body +passed diff --git a/t/plugin/log-rotate2.t b/t/plugin/log-rotate2.t index d611ffc..b04a6f5 100644 --- a/t/plugin/log-rotate2.t +++ b/t/plugin/log-rotate2.t @@ -41,7 +41,15 @@ plugin_attr: _EOC_ $block->set_value("yaml_config", $user_yaml_config); - $block->set_value("request", "GET /t"); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + }); run_tests; @@ -74,8 +82,6 @@ __DATA__ end } } ---- no_error_log -[error] @@ -90,7 +96,45 @@ __DATA__ } --- response_body done ---- no_error_log -[error] --- error_log start xxxxxx + + + +=== TEST 3: check file changes (enable compression) +--- config + location /t { + content_by_lua_block { + ngx.sleep(2) + + local default_logs = {} + for file_name in lfs.dir(ngx.config.prefix() .. "/logs/") do + if string.match(file_name, "__error.log.tar.gz$") or string.match(file_name, "__access.log.tar.gz$") then + local filepath = ngx.config.prefix() .. "/logs/" .. file_name + local attr = lfs.attributes(filepath) + if attr then + default_logs[filepath] = { change = attr.change, size = attr.size } + end + end + end + + ngx.sleep(1) + + local passed = false + for filepath, origin_attr in pairs(default_logs) do + local check_attr = lfs.attributes(filepath) + if check_attr.change == origin_attr.change and check_attr.size == origin_attr.size then + passed = true + else + passed = false + break + end + end + + if passed then + ngx.say("passed") + end + } + } +--- response_body +passed