This is an automated email from the ASF dual-hosted git repository. mani pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new d02c41d5 [YUNIKORN-2119] check wildcard limit in different queue level
(#720)
d02c41d5 is described below
commit d02c41d519189b9083802a68e5aec556db2d2304
Author: PoAn Yang <pay...@apache.org>
AuthorDate: Mon Dec 18 11:17:29 2023 +0530
[YUNIKORN-2119] check wildcard limit in different queue level (#720)
Closes: #720
Signed-off-by: Manikandan R <maniraj...@gmail.com>
---
pkg/common/configs/configvalidator.go | 10 ++
pkg/common/configs/configvalidator_test.go | 241 ++++++++++++++++++++++++++---
2 files changed, 232 insertions(+), 19 deletions(-)
diff --git a/pkg/common/configs/configvalidator.go
b/pkg/common/configs/configvalidator.go
index bd12d535..e9e28168 100644
--- a/pkg/common/configs/configvalidator.go
+++ b/pkg/common/configs/configvalidator.go
@@ -172,6 +172,11 @@ func checkLimitResource(cur QueueConfig, users
map[string]map[string]*resources.
}
// Override with min resource
users[curQueuePath][user] =
resources.ComponentWiseMinPermissive(limitMaxResources, userMaxResource)
+ } else if wildcardMaxResource, ok :=
users[queuePath][common.Wildcard]; user != common.Wildcard && ok {
+ if
!wildcardMaxResource.FitInMaxUndef(limitMaxResources) {
+ return fmt.Errorf("user %s max resource
%s of queue %s is greater than wildcard maximum resource %s of immediate or
ancestor parent queue", user, limitMaxResources.String(), cur.Name,
wildcardMaxResource.String())
+ }
+ users[curQueuePath][user] = limitMaxResources
} else {
users[curQueuePath][user] = limitMaxResources
}
@@ -184,6 +189,11 @@ func checkLimitResource(cur QueueConfig, users
map[string]map[string]*resources.
}
// Override with min resource
groups[curQueuePath][group] =
resources.ComponentWiseMinPermissive(limitMaxResources, groupMaxResource)
+ } else if wildcardMaxResource, ok :=
groups[queuePath][common.Wildcard]; group != common.Wildcard && ok {
+ if
!wildcardMaxResource.FitInMaxUndef(limitMaxResources) {
+ return fmt.Errorf("group %s max
resource %s of queue %s is greater than wildcard maximum resource %s of
immediate or ancestor parent queue", group, limitMaxResources.String(),
cur.Name, wildcardMaxResource.String())
+ }
+ groups[curQueuePath][group] = limitMaxResources
} else {
groups[curQueuePath][group] = limitMaxResources
}
diff --git a/pkg/common/configs/configvalidator_test.go
b/pkg/common/configs/configvalidator_test.go
index 6a87f933..b8079b3d 100644
--- a/pkg/common/configs/configvalidator_test.go
+++ b/pkg/common/configs/configvalidator_test.go
@@ -740,32 +740,34 @@ func TestServiceAccountGroupName(t *testing.T) {
func TestCheckLimitResource(t *testing.T) { //nolint:funlen
testCases := []struct {
- name string
- config QueueConfig
- hasError bool
+ name string
+ config QueueConfig
+ errMsg string
}{
{
name: "leaf queue user group maxresources are within
immediate parent queue user group maxresources",
config: QueueConfig{
Name: "parent",
- Limits:
createLimitMaxResources(map[string]map[string]string{"test-user": {"memory":
"100"}},
-
map[string]map[string]string{"test-group": {"memory": "100"}}),
+ Limits: createLimitMaxResources(
+
map[string]map[string]string{"test-user": {"memory": "100"}, "*": {"memory":
"100"}},
+
map[string]map[string]string{"test-group": {"memory": "100"}, "*": {"memory":
"100"}}),
Queues: []QueueConfig{
{
Name: "child1",
- Limits:
createLimitMaxResources(map[string]map[string]string{"test-user": {"memory":
"50"}},
-
map[string]map[string]string{"test-group": {"memory": "50"}}),
+ Limits: createLimitMaxResources(
+
map[string]map[string]string{"test-user": {"memory": "50"}, "test-user2":
{"memory": "50"}},
+
map[string]map[string]string{"test-group": {"memory": "50"}, "test-group2":
{"memory": "50"}}),
Queues: []QueueConfig{
{
Name: "child2",
- Limits:
createLimitMaxResources(map[string]map[string]string{"test-user": {"memory":
"10"}},
-
map[string]map[string]string{"test-group": {"memory": "10"}}),
+ Limits:
createLimitMaxResources(
+
map[string]map[string]string{"test-user": {"memory": "10"}, "test-user2":
{"memory": "10"}},
+
map[string]map[string]string{"test-group": {"memory": "10"}, "test-group2":
{"memory": "10"}}),
},
},
},
},
},
- hasError: false,
},
{
name: "leaf queue user group maxresources are within
ancestor parent queue user group maxresources",
@@ -791,7 +793,6 @@ func TestCheckLimitResource(t *testing.T) { //nolint:funlen
},
},
},
- hasError: false,
},
{
name: "leaf queue user maxresources exceed parent queue
user maxresources",
@@ -807,7 +808,7 @@ func TestCheckLimitResource(t *testing.T) { //nolint:funlen
},
},
},
- hasError: true,
+ errMsg: "is greater than immediate or ancestor parent
maximum resource",
},
{
name: "leaf queue group maxresources exceed parent
queue group maxresources",
@@ -823,7 +824,7 @@ func TestCheckLimitResource(t *testing.T) { //nolint:funlen
},
},
},
- hasError: true,
+ errMsg: "is greater than immediate or ancestor parent
maximum resource",
},
{
name: "queues at same level maxresources can be greater
or less than or equal to the other but with in immediate parent",
@@ -844,7 +845,6 @@ func TestCheckLimitResource(t *testing.T) { //nolint:funlen
},
},
},
- hasError: false,
},
{
name: "queues at same level maxresources can be greater
or less than or equal to the other but with in ancestor parent",
@@ -870,7 +870,6 @@ func TestCheckLimitResource(t *testing.T) { //nolint:funlen
},
},
},
- hasError: false,
},
{
name: "leaf queue user maxresources exceed grandparent
queue user maxresources",
@@ -904,7 +903,7 @@ func TestCheckLimitResource(t *testing.T) { //nolint:funlen
},
},
},
- hasError: true,
+ errMsg: "is greater than immediate or ancestor parent
maximum resource",
},
{
name: "leaf queue group maxresources exceed grandparent
queue group maxresources",
@@ -938,15 +937,219 @@ func TestCheckLimitResource(t *testing.T) {
//nolint:funlen
},
},
},
- hasError: true,
+ errMsg: "is greater than immediate or ancestor parent
maximum resource",
+ },
+ {
+ name: "leaf queue user maxresources exceed parent queue
wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ map[string]map[string]string{
+ "test-user1": {"memory": "100"},
+ "*": {"memory": "100"},
+ },
+ nil),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Limits: createLimitMaxResources(
+
map[string]map[string]string{
+ "test-user1":
{"memory": "50"},
+ "test-user2":
{"memory": "150"},
+ },
+ nil),
+ },
+ },
+ },
+ errMsg: "is greater than wildcard maximum resource",
+ },
+ {
+ name: "leaf queue group maxresources exceed parent
queue wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ nil,
+ map[string]map[string]string{
+ "test-group1": {"memory":
"100"},
+ "*": {"memory":
"100"},
+ }),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Limits: createLimitMaxResources(
+ nil,
+
map[string]map[string]string{
+ "test-group1":
{"memory": "50"},
+ "test-group2":
{"memory": "150"},
+ }),
+ },
+ },
+ },
+ errMsg: "is greater than wildcard maximum resource",
+ },
+ {
+ name: "leaf queue user maxresources exceed grandparent
queue wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ map[string]map[string]string{
+ "test-user1": {"memory": "100"},
+ "*": {"memory": "100"},
+ },
+ nil),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Limits: createLimitMaxResources(
+
map[string]map[string]string{
+ "test-user1":
{"memory": "50"},
+ },
+ nil),
+ Queues: []QueueConfig{
+ {
+ Name: "child2",
+ Limits:
createLimitMaxResources(
+
map[string]map[string]string{
+
"test-user1": {"memory": "10"},
+
"test-user2": {"memory": "150"},
+ },
+ nil),
+ },
+ },
+ },
+ },
+ },
+ errMsg: "is greater than wildcard maximum resource",
+ },
+ {
+ name: "leaf queue group maxresources exceed grandparent
queue wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ nil,
+ map[string]map[string]string{
+ "test-group1": {"memory":
"100"},
+ "*": {"memory":
"100"},
+ }),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Limits: createLimitMaxResources(
+ nil,
+
map[string]map[string]string{
+ "test-group1":
{"memory": "50"},
+ }),
+ Queues: []QueueConfig{
+ {
+ Name: "child2",
+ Limits:
createLimitMaxResources(
+ nil,
+
map[string]map[string]string{
+
"test-group1": {"memory": "10"},
+
"test-group2": {"memory": "150"},
+ }),
+ },
+ },
+ },
+ },
+ },
+ errMsg: "is greater than wildcard maximum resource",
+ },
+ {
+ name: "leaf queue user wildcard maxresources are within
immediate parent queue wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ map[string]map[string]string{"*":
{"memory": "100"}},
+ map[string]map[string]string{}),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Limits: createLimitMaxResources(
+
map[string]map[string]string{"*": {"memory": "50"}},
+
map[string]map[string]string{}),
+ Queues: []QueueConfig{
+ {
+ Name: "child2",
+ Limits:
createLimitMaxResources(
+
map[string]map[string]string{"*": {"memory": "10"}},
+
map[string]map[string]string{}),
+ },
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "leaf queue user wildcard maxresources are within
grandparent queue wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ map[string]map[string]string{"*":
{"memory": "100"}},
+ map[string]map[string]string{}),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Queues: []QueueConfig{
+ {
+ Name: "child2",
+ Limits:
createLimitMaxResources(
+
map[string]map[string]string{"*": {"memory": "10"}},
+
map[string]map[string]string{}),
+ },
+ },
+ },
+ },
+ },
+ },
+ {
+ name: "leaf queue user wildcard maxresources exceed
immediate parent queue wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ map[string]map[string]string{"*":
{"memory": "100"}},
+ map[string]map[string]string{}),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Limits: createLimitMaxResources(
+
map[string]map[string]string{"*": {"memory": "150"}},
+
map[string]map[string]string{}),
+ },
+ },
+ },
+ errMsg: "is greater than immediate or ancestor parent
maximum resource",
+ },
+ {
+ name: "leaf queue user wildcard maxresources exceed
grandparent queue wildcard maxresources",
+ config: QueueConfig{
+ Name: "parent",
+ Limits: createLimitMaxResources(
+ map[string]map[string]string{"*":
{"memory": "100"}},
+ map[string]map[string]string{}),
+ Queues: []QueueConfig{
+ {
+ Name: "child1",
+ Queues: []QueueConfig{
+ {
+ Name: "child2",
+ Limits:
createLimitMaxResources(
+
map[string]map[string]string{"*": {"memory": "150"}},
+
map[string]map[string]string{}),
+ },
+ },
+ },
+ },
+ },
+ errMsg: "is greater than immediate or ancestor parent
maximum resource",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := checkLimitResource(testCase.config,
make(map[string]map[string]*resources.Resource),
make(map[string]map[string]*resources.Resource), common.Empty)
- if testCase.hasError {
- assert.ErrorContains(t, err, "is greater than
immediate or ancestor parent maximum resource")
+ if testCase.errMsg != "" {
+ assert.ErrorContains(t, err, testCase.errMsg)
} else {
assert.NilError(t, err)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@yunikorn.apache.org
For additional commands, e-mail: issues-h...@yunikorn.apache.org
