This is an automated email from the ASF dual-hosted git repository. maxyang pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit 8369cf6dff784bc0164f58ba98112fcfce8a4412 Author: RMT <[email protected]> AuthorDate: Wed Sep 6 20:04:05 2023 -0600 Improve io_limit behaviors (#16351) * add function to clear io.max This pr has several improvements for io limit: 1. Add a function to clear io.max. This function should be used when alter io_limit. 2. Check tablespace in io_limit when drop tablespaces. If the tablespace which will be dropped presents in some io_limit resource groups, the drop tablespace statement will be aborted. 3. When InitResGroup and AlterResourceGroup, if parseio raises an error, the error will be demote to WARNING. So the cluster can launch when some tablespace has been removed. --- src/backend/commands/resgroupcmds.c | 29 ++++++- src/backend/commands/tablespace.c | 5 ++ src/backend/utils/resgroup/cgroup-ops-dummy.c | 10 ++- src/backend/utils/resgroup/cgroup-ops-linux-v1.c | 12 ++- src/backend/utils/resgroup/cgroup-ops-linux-v2.c | 10 ++- src/backend/utils/resgroup/cgroup_io_limit.c | 59 ++++++++++++- src/backend/utils/resgroup/resgroup.c | 96 +++++++++++++++++++++- src/include/utils/cgroup.h | 2 + src/include/utils/cgroup_io_limit.h | 6 +- src/include/utils/resgroup.h | 1 + .../resgroup/resgroup_auxiliary_tools_v2.out | 8 ++ .../input/resgroup/resgroup_io_limit.source | 5 ++ .../output/resgroup/resgroup_io_limit.source | 11 +++ .../sql/resgroup/resgroup_auxiliary_tools_v2.sql | 19 +++++ 14 files changed, 265 insertions(+), 8 deletions(-) diff --git a/src/backend/commands/resgroupcmds.c b/src/backend/commands/resgroupcmds.c index 5c319e85b11..ce5ab6d584f 100644 --- a/src/backend/commands/resgroupcmds.c +++ b/src/backend/commands/resgroupcmds.c @@ -552,6 +552,8 @@ AlterResourceGroup(AlterResourceGroupStmt *stmt) /* * Get all the capabilities of one resource group in pg_resgroupcapability. + * + * Note: the io_limit in ResGroupCaps will be NIL if parse io_limit string failed. */ void GetResGroupCapabilities(Relation rel, Oid groupId, ResGroupCaps *resgroupCaps) @@ -634,9 +636,34 @@ GetResGroupCapabilities(Relation rel, Oid groupId, ResGroupCaps *resgroupCaps) case RESGROUP_LIMIT_TYPE_IO_LIMIT: if (cgroupOpsRoutine != NULL) { + /* if InterruptHoldoffCount doesn't restore in PG_CATCH, + * the catch will failed. */ + int32 savedholdoffCount = InterruptHoldoffCount; + oldContext = CurrentMemoryContext; MemoryContextSwitchTo(TopMemoryContext); - resgroupCaps->io_limit = cgroupOpsRoutine->parseio(value); + /* + * This function will be called in InitResGroups and AlterResourceGroup which + * shoud not be abort. In some circumstances, for example, the directory of a + * tablespace in io_limit be removed, then parseio will throw error. If + * InitResGroups be aborted, the cluster cannot launched. */ + PG_TRY(); + { + resgroupCaps->io_limit = cgroupOpsRoutine->parseio(value); + } + PG_CATCH(); + { + resgroupCaps->io_limit = NIL; + + InterruptHoldoffCount = savedholdoffCount; + + if (elog_demote(WARNING)) + { + EmitErrorReport(); + FlushErrorState(); + } + } + PG_END_TRY(); MemoryContextSwitchTo(oldContext); } else diff --git a/src/backend/commands/tablespace.c b/src/backend/commands/tablespace.c index ef8bd610e0b..9175ebfb5ba 100644 --- a/src/backend/commands/tablespace.c +++ b/src/backend/commands/tablespace.c @@ -97,8 +97,10 @@ #include "utils/lsyscache.h" #include "utils/memutils.h" #include "utils/rel.h" +#include "utils/resource_manager.h" #include "utils/tarrable.h" #include "utils/varlena.h" +#include "utils/resgroup.h" #include "catalog/heap.h" #include "catalog/oid_dispatch.h" @@ -738,6 +740,9 @@ DropTableSpace(DropTableSpaceStmt *stmt) errdetail_internal("%s", detail), errdetail_log("%s", detail_log))); + if (IsResGroupEnabled() && Gp_resource_manager_policy == RESOURCE_MANAGER_POLICY_GROUP_V2) + checkTablespaceInIOlimit(tablespaceoid, true); + /* DROP hook for the tablespace being removed */ InvokeObjectDropHook(TableSpaceRelationId, tablespaceoid, 0); diff --git a/src/backend/utils/resgroup/cgroup-ops-dummy.c b/src/backend/utils/resgroup/cgroup-ops-dummy.c index 9c5c2850aea..91c588f5986 100644 --- a/src/backend/utils/resgroup/cgroup-ops-dummy.c +++ b/src/backend/utils/resgroup/cgroup-ops-dummy.c @@ -252,6 +252,12 @@ dumpio_dummy(List *limit_list) return DefaultIOLimit; } +static void +cleario_dummy(Oid groupid) +{ + unsupported_system(); +} + static CGroupOpsRoutine cGroupOpsRoutineDummy = { .getcgroupname = getcgroupname_dummy, @@ -280,7 +286,9 @@ static CGroupOpsRoutine cGroupOpsRoutineDummy = { .setio = setio_dummy, .freeio = freeio_dummy, .getiostat = getiostat_dummy, - .dumpio = dumpio_dummy + .dumpio = dumpio_dummy, + .cleario = cleario_dummy + }; CGroupOpsRoutine *get_cgroup_routine_dummy(void) diff --git a/src/backend/utils/resgroup/cgroup-ops-linux-v1.c b/src/backend/utils/resgroup/cgroup-ops-linux-v1.c index 96390712be4..fffba070a3f 100644 --- a/src/backend/utils/resgroup/cgroup-ops-linux-v1.c +++ b/src/backend/utils/resgroup/cgroup-ops-linux-v1.c @@ -176,6 +176,7 @@ static void setio_v1(Oid group, List *limit_list); static void freeio_v1(List *limit_list); static List* getiostat_v1(Oid group, List *io_limit); static char *dumpio_v1(List *limit_list); +static void cleario_v1(Oid groupid); /* * Detect gpdb cgroup component dirs. @@ -1158,6 +1159,14 @@ dumpio_v1(List *limit_list) return DefaultIOLimit; } +static void +cleario_v1(Oid groupid) +{ + ereport(WARNING, + (errcode(ERRCODE_SYSTEM_ERROR), + errmsg("resource group io limit only can be used in cgroup v2."))); +} + static CGroupOpsRoutine cGroupOpsRoutineV1 = { .getcgroupname = getcgroupname_v1, .probecgroup = probecgroup_v1, @@ -1187,7 +1196,8 @@ static CGroupOpsRoutine cGroupOpsRoutineV1 = { .setio = setio_v1, .freeio = freeio_v1, .getiostat = getiostat_v1, - .dumpio = dumpio_v1 + .dumpio = dumpio_v1, + .cleario = cleario_v1 }; CGroupOpsRoutine *get_group_routine_v1(void) diff --git a/src/backend/utils/resgroup/cgroup-ops-linux-v2.c b/src/backend/utils/resgroup/cgroup-ops-linux-v2.c index 546b2272213..933f8b139ef 100644 --- a/src/backend/utils/resgroup/cgroup-ops-linux-v2.c +++ b/src/backend/utils/resgroup/cgroup-ops-linux-v2.c @@ -166,6 +166,7 @@ static void setio_v2(Oid group, List *limit_list); static void freeio_v2(List *limit_list); static List *getiostat_v2(Oid group, List *io_limit); static char *dumpio_v2(List *limit_list); +static void cleario_v2(Oid groupid); /* * Dump component dir to the log. @@ -906,6 +907,12 @@ dumpio_v2(List *limit_list) return io_limit_dump(limit_list); } +static void +cleario_v2(Oid groupid) +{ + clear_io_max(groupid); +} + static CGroupOpsRoutine cGroupOpsRoutineV2 = { .getcgroupname = getcgroupname_v2, .probecgroup = probecgroup_v2, @@ -935,7 +942,8 @@ static CGroupOpsRoutine cGroupOpsRoutineV2 = { .setio = setio_v2, .freeio = freeio_v2, .getiostat = getiostat_v2, - .dumpio = dumpio_v2 + .dumpio = dumpio_v2, + .cleario = cleario_v2 }; CGroupOpsRoutine *get_group_routine_v2(void) diff --git a/src/backend/utils/resgroup/cgroup_io_limit.c b/src/backend/utils/resgroup/cgroup_io_limit.c index f3b1a1e33b2..29737a25ff8 100644 --- a/src/backend/utils/resgroup/cgroup_io_limit.c +++ b/src/backend/utils/resgroup/cgroup_io_limit.c @@ -1,5 +1,4 @@ #include "postgres.h" -#include "utils/cgroup_io_limit.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup.h" @@ -9,6 +8,8 @@ #include "catalog/indexing.h" #include "catalog/pg_resgroup_d.h" #include "catalog/pg_resgroupcapability_d.h" +#include "common/string.h" +#include "commands/tablespace.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_tablespace_d.h" #include "commands/resgroupcmds.h" @@ -18,11 +19,18 @@ #include "port.h" #include "storage/fd.h" #include "utils/cgroup.h" +#include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/hsearch.h" #include "utils/palloc.h" #include "utils/relcache.h" #include "utils/resgroup.h" +#include "utils/cgroup_io_limit.h" +#include <utils/cgroup.h> + +#ifndef __linux__ +#error cgroup is only available on linux +#endif #include <libgen.h> #include <limits.h> @@ -536,7 +544,10 @@ io_limit_dump(List *limit_list) for(i = 0; i < fields_length; i++) { - appendStringInfo(result, "%s=%lu", IOconfigFields[i], *(value + i)); + if (*(value + i) != IO_LIMIT_MAX) + appendStringInfo(result, "%s=%lu", IOconfigFields[i], *(value + i)); + else + appendStringInfo(result, "%s=max", IOconfigFields[i]); if (i + 1 != fields_length) appendStringInfo(result, ","); @@ -548,3 +559,47 @@ io_limit_dump(List *limit_list) return result->data; } + +void +clear_io_max(Oid groupid) +{ + FILE *f; + StringInfo line = makeStringInfo(); + StringInfo result = makeStringInfo(); + List *result_lines = NIL; + ListCell *cell; + char path[MAX_CGROUP_PATHLEN]; + buildPath(groupid, BASEDIR_GPDB, CGROUP_COMPONENT_PLAIN, "io.max", path, MAX_CGROUP_PATHLEN); + + f = AllocateFile(path, "r"); + /* pg_get_line_buf will reset line each time */ + while (pg_get_line_buf(f, line)) + { + uint32 maj, min; + int i; + char *str = line->data; + + sscanf(str, "%u:%u", &maj, &min); + + appendStringInfo(result, "%u:%u", maj, min); + for (i = 0; i < lengthof(IOconfigFields); i++) + appendStringInfo(result, " %s=max", IOconfigFields[i]); + + result_lines = lappend(result_lines, (void *) pstrdup(result->data)); + + resetStringInfo(result); + } + FreeFile(f); + pfree(line->data); + pfree(result->data); + pfree(line); + pfree(result); + + foreach(cell, result_lines) + { + char *str = (char *)lfirst(cell); + writeStr(groupid, BASEDIR_GPDB, CGROUP_COMPONENT_PLAIN, "io.max", str); + } + + list_free_deep(result_lines); +} diff --git a/src/backend/utils/resgroup/resgroup.c b/src/backend/utils/resgroup/resgroup.c index 6dc5f0c8174..4a6c9c14b94 100644 --- a/src/backend/utils/resgroup/resgroup.c +++ b/src/backend/utils/resgroup/resgroup.c @@ -33,6 +33,7 @@ #include "cdb/cdbdisp_query.h" #include "cdb/memquota.h" #include "commands/resgroupcmds.h" +#include "commands/tablespace.h" #include "common/hashfn.h" #include "funcapi.h" #include "miscadmin.h" @@ -59,6 +60,7 @@ #include "utils/cgroup-ops-v1.h" #include "utils/cgroup-ops-dummy.h" #include "utils/cgroup-ops-v2.h" +#include "utils/cgroup_io_limit.h" #include "access/xact.h" #define InvalidSlotId (-1) @@ -805,7 +807,11 @@ ResGroupAlterOnCommit(const ResourceGroupCallbackContext *callbackCtx) } else if (callbackCtx->limittype == RESGROUP_LIMIT_TYPE_IO_LIMIT) { - cgroupOpsRoutine->setio(callbackCtx->groupid, callbackCtx->caps.io_limit); + if (callbackCtx->caps.io_limit != NIL) + { + cgroupOpsRoutine->cleario(callbackCtx->groupid); + cgroupOpsRoutine->setio(callbackCtx->groupid, callbackCtx->caps.io_limit); + } } /* reset default group if cpuset has changed */ @@ -3701,6 +3707,94 @@ check_and_unassign_from_resgroup(PlannedStmt* stmt) bypassedGroup->caps.cpuMaxPercent == CPU_MAX_PERCENT_DISABLED); } +/* + * return ture if there is a resource group which io_limit contains tblspcid. + * if errout is true, print warning, message. */ +bool +checkTablespaceInIOlimit(Oid tblspcid, bool errout) +{ + Relation rel_resgroup_caps; + SysScanDesc sscan; + HeapTuple tuple; + bool contain = false; + bool print_header = false; + StringInfo log = makeStringInfo(); + + rel_resgroup_caps = table_open(ResGroupCapabilityRelationId, AccessShareLock); + /* get io limit string from catalog */ + sscan = systable_beginscan(rel_resgroup_caps, InvalidOid, false, + NULL, 0, NULL); + while (HeapTupleIsValid(tuple = systable_getnext(sscan))) + { + Oid id; + bool isNULL; + Datum id_datum; + Datum type_datum; + Datum value_datum; + ResGroupLimitType type; + char *io_limit_str; + List *limit_list; + ListCell *cell; + + type_datum = heap_getattr(tuple, Anum_pg_resgroupcapability_reslimittype, + rel_resgroup_caps->rd_att, &isNULL); + type = (ResGroupLimitType) DatumGetInt16(type_datum); + if (type != RESGROUP_LIMIT_TYPE_IO_LIMIT) + continue; + + id_datum = heap_getattr(tuple, Anum_pg_resgroupcapability_resgroupid, + rel_resgroup_caps->rd_att, &isNULL); + id = DatumGetObjectId(id_datum); + + value_datum = heap_getattr(tuple, Anum_pg_resgroupcapability_value, + rel_resgroup_caps->rd_att, &isNULL); + io_limit_str = TextDatumGetCString(value_datum); + + if (strcmp(io_limit_str, DefaultIOLimit) == 0) + continue; + + limit_list = cgroupOpsRoutine->parseio(io_limit_str); + foreach(cell, limit_list) + { + TblSpcIOLimit *limit = (TblSpcIOLimit *) lfirst(cell); + + if (limit->tablespace_oid == tblspcid) + { + contain = true; + + if (!errout) + break; + + if (!print_header) + { + print_header = true; + appendStringInfo(log, "io limit: following resource groups depend on tablespace %s:", get_tablespace_name(tblspcid)); + } + + appendStringInfo(log, " %s", GetResGroupNameForId(id)); + } + } + + cgroupOpsRoutine->freeio(limit_list); + + if (contain && !errout) + break; + } + systable_endscan(sscan); + + table_close(rel_resgroup_caps, AccessShareLock); + + if (contain && errout) + ereport(ERROR, (errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST), + errmsg("%s", log->data), + errhint("you can remove those resource groups or remove tablespace %s from io_limit of those resource groups.", get_tablespace_name(tblspcid)))); + + pfree(log->data); + pfree(log); + + return contain; +} + /* * Given a planned statement, check if it is pure catalog query or a very simple query. * Return true only when: diff --git a/src/include/utils/cgroup.h b/src/include/utils/cgroup.h index 42e6468be40..b9c0fa249a2 100644 --- a/src/include/utils/cgroup.h +++ b/src/include/utils/cgroup.h @@ -233,6 +233,7 @@ typedef void (*setio_function) (Oid group, List *limit_list); typedef void (*freeio_function) (List *limit_list); typedef List* (*getiostat_function) (Oid groupid, List *io_limit); typedef char* (*dumpio_function) (List *limit_list); +typedef void (*cleario_function) (Oid groupid); typedef struct CGroupOpsRoutine @@ -272,6 +273,7 @@ typedef struct CGroupOpsRoutine freeio_function freeio; getiostat_function getiostat; dumpio_function dumpio; + cleario_function cleario; } CGroupOpsRoutine; /* The global function handler. */ diff --git a/src/include/utils/cgroup_io_limit.h b/src/include/utils/cgroup_io_limit.h index e7ea1528899..b10482e16d8 100644 --- a/src/include/utils/cgroup_io_limit.h +++ b/src/include/utils/cgroup_io_limit.h @@ -7,11 +7,14 @@ /* type for linux device id, use libc dev_t now. * bdi means backing device info. */ +#ifdef __linux__ typedef dev_t bdi_t; - #define make_bdi(major, minor) makedev(major, minor) #define bdi_major(bdi) major(bdi) #define bdi_minor(bdi) minor(bdi) +#else +typedef uint64 bdi_t; +#endif #define IO_LIMIT_MAX (0) @@ -108,5 +111,6 @@ extern void io_limit_validate(List *limit_list); extern List *get_iostat(Oid groupid, List *io_limit); extern int compare_iostat(const ListCell *ca, const ListCell *cb); extern char *io_limit_dump(List *limit_list); +extern void clear_io_max(Oid groupid); #endif diff --git a/src/include/utils/resgroup.h b/src/include/utils/resgroup.h index 6db5a10a0e8..035fc180161 100644 --- a/src/include/utils/resgroup.h +++ b/src/include/utils/resgroup.h @@ -216,6 +216,7 @@ extern void ResGroupMoveQuery(int sessionId, Oid groupId, const char *groupName) extern Oid ResGroupGetGroupIdBySessionId(int sessionId); extern char *getCpuSetByRole(const char *cpuset); extern void checkCpuSetByRole(const char *cpuset); +extern bool checkTablespaceInIOlimit(Oid tblspcid, bool errout); #define LOG_RESGROUP_DEBUG(...) \ do {if (Debug_resource_group) elog(__VA_ARGS__); } while(false); diff --git a/src/test/isolation2/expected/resgroup/resgroup_auxiliary_tools_v2.out b/src/test/isolation2/expected/resgroup/resgroup_auxiliary_tools_v2.out index 45bd1cb45e1..61e1e7474c2 100644 --- a/src/test/isolation2/expected/resgroup/resgroup_auxiliary_tools_v2.out +++ b/src/test/isolation2/expected/resgroup/resgroup_auxiliary_tools_v2.out @@ -147,3 +147,11 @@ fcntl.flock(f, fcntl.LOCK_EX) if not os.path.exists(dirname): os.close(f) return True try: shutil.rmtree(dirname) except Exception as e: plpy.error("cannot remove dir {}".format(e)) else: return True finally: os.close(f) $$ LANGUAGE plpython3u; CREATE + +0: CREATE OR REPLACE FUNCTION check_clear_io_max(groupname text) RETURNS BOOL AS $$ import ctypes import os +postgres = ctypes.CDLL(None) clear_io_max = postgres['clear_io_max'] +# get group oid sql = "select groupid from gp_toolkit.gp_resgroup_config where groupname = '%s'" % groupname result = plpy.execute(sql) groupid = result[0]['groupid'] +clear_io_max(groupid) +cgroup_path = "/sys/fs/cgroup/gpdb/%d/io.max" % groupid +return os.stat(cgroup_path).st_size == 0 $$ LANGUAGE plpython3u; +CREATE diff --git a/src/test/isolation2/input/resgroup/resgroup_io_limit.source b/src/test/isolation2/input/resgroup/resgroup_io_limit.source index d3a0bd485a0..e2e14a7b7a5 100644 --- a/src/test/isolation2/input/resgroup/resgroup_io_limit.source +++ b/src/test/isolation2/input/resgroup/resgroup_io_limit.source @@ -44,10 +44,15 @@ CREATE RESOURCE GROUP rg_test_group5 WITH (concurrency=10, cpu_max_percent=10, i SELECT check_cgroup_io_max('rg_test_group5', 'rg_io_limit_ts_1', 'rbps=1048576000 wbps=1048576000 riops=1000 wiops=1000'); +-- try to drop test tablespace +DROP TABLESPACE rg_io_limit_ts_1; + ALTER RESOURCE GROUP rg_test_group5 SET io_limit 'rg_io_limit_ts_1:rbps=1000,wbps=1000'; SELECT check_cgroup_io_max('rg_test_group5', 'rg_io_limit_ts_1', 'rbps=1048576000 wbps=1048576000 riops=max wiops=max'); +SELECT check_clear_io_max('rg_test_group5'); + -- fault injector SELECT gp_inject_fault('create_resource_group_fail', 'error', 1); diff --git a/src/test/isolation2/output/resgroup/resgroup_io_limit.source b/src/test/isolation2/output/resgroup/resgroup_io_limit.source index 9e40b8c34d9..4c2d90c163c 100644 --- a/src/test/isolation2/output/resgroup/resgroup_io_limit.source +++ b/src/test/isolation2/output/resgroup/resgroup_io_limit.source @@ -80,6 +80,11 @@ SELECT check_cgroup_io_max('rg_test_group5', 'rg_io_limit_ts_1', 'rbps=104857600 t (1 row) +-- try to drop test tablespace +DROP TABLESPACE rg_io_limit_ts_1; +ERROR: io limit: following resource groups depend on tablespace rg_io_limit_ts_1: rg_test_group5 +HINT: you can remove those resource groups or remove tablespace rg_io_limit_ts_1 from io_limit of those resource groups. + ALTER RESOURCE GROUP rg_test_group5 SET io_limit 'rg_io_limit_ts_1:rbps=1000,wbps=1000'; ALTER @@ -89,6 +94,12 @@ SELECT check_cgroup_io_max('rg_test_group5', 'rg_io_limit_ts_1', 'rbps=104857600 t (1 row) +SELECT check_clear_io_max('rg_test_group5'); + check_clear_io_max +-------------------- + t +(1 row) + -- fault injector SELECT gp_inject_fault('create_resource_group_fail', 'error', 1); gp_inject_fault diff --git a/src/test/isolation2/sql/resgroup/resgroup_auxiliary_tools_v2.sql b/src/test/isolation2/sql/resgroup/resgroup_auxiliary_tools_v2.sql index 966d121a5dc..6a60a32e3f7 100644 --- a/src/test/isolation2/sql/resgroup/resgroup_auxiliary_tools_v2.sql +++ b/src/test/isolation2/sql/resgroup/resgroup_auxiliary_tools_v2.sql @@ -351,3 +351,22 @@ $$ LANGUAGE plpython3u; finally: os.close(f) $$ LANGUAGE plpython3u; + +0: CREATE OR REPLACE FUNCTION check_clear_io_max(groupname text) RETURNS BOOL AS $$ + import ctypes + import os + + postgres = ctypes.CDLL(None) + clear_io_max = postgres['clear_io_max'] + + # get group oid + sql = "select groupid from gp_toolkit.gp_resgroup_config where groupname = '%s'" % groupname + result = plpy.execute(sql) + groupid = result[0]['groupid'] + + clear_io_max(groupid) + + cgroup_path = "/sys/fs/cgroup/gpdb/%d/io.max" % groupid + + return os.stat(cgroup_path).st_size == 0 +$$ LANGUAGE plpython3u; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
