On Tue, 30 Jun 2015 at 09:28 -0000, Alexis Huxley wrote:
> Hi, is there any news regarding cgroup-based enforcement of h_vmem?
We also wanted this. Our primary use case was users needing to mmap()
large files.
> I saw that Mark Dixon posted patches against 8.1.1 (details at
> http://gridengine.org/pipermail/users/2012-September/004699.html),
> but these are not in 8.1.8 nor the 20150129 snapshot.
There was also a followup email with updated patches:
http://arc.liv.ac.uk/pipermail/sge-discuss/2012-December/000343.html
> We would like to use exactly this facility, but on 8.1.8.
>
> Before I try to port Mark's patches, I thought it made sense to ask
> here in case "native" support is just around the corner, or somebody
> else has already done this porting, or somebody has written their
> own patches. Thanks!
We have applied parts of Mark's patch set to our 8.1.8 system. The
consolidated patches we use are attached. I don't believe we did
anything significant except update them for 8.1.8 sources.
We have been running with these patches for a few months now.
I think there might be some interactions with cgroup based cpuset/core
binding support which will eventually need addressing. We haven't
looked at that yet.
Stuart Barkley
--
I've never been lost; I was once bewildered for three days, but never lost!
-- Daniel Boonediff -ruN sge-8.1.8-orig/doc/man/man5/sge_conf.5
sge-8.1.8/doc/man/man5/sge_conf.5
--- sge-8.1.8-orig/doc/man/man5/sge_conf.5 2014-10-23 18:03:54.000000000
-0400
+++ sge-8.1.8/doc/man/man5/sge_conf.5 2015-03-31 14:18:00.010993411 -0400
@@ -1559,6 +1559,11 @@
.I sources/experimental/qidle
in the source repository, but it may be better to check the
the screensaver state.
+.IP \fICGROUP_MEMORY\fP
+If this parameter is set to a directory, then xxQS_NAMExx will create a
+cgroup in that location for each job instance running on the execd. It
+will use this to control the amount of memory (both rss and vmem) used
+by jobs.
.IP \fIUSE_CGROUPS\fP
[Linux only.] Use cgroups/cpusets for resource management if the
system supports them and the necessary directories exist in the
diff -ruN sge-8.1.8-orig/source/daemons/common/Makefile
sge-8.1.8/source/daemons/common/Makefile
--- sge-8.1.8-orig/source/daemons/common/Makefile 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/common/Makefile 2015-03-31 14:18:00.035178000
-0400
@@ -48,6 +48,7 @@
sge_processes_irix.c \
pdc.c \
procfs.c \
+ cgroup.c \
binding_support.c
DAEMONS_COMMON_TEST_OBJS = \
@@ -148,6 +149,9 @@
procfs.o: $(DAEMONS_COMMON_DIR)/procfs.c
$(CC) $(CFLAGS) $(MEMCFLAG) -c $(DAEMONS_COMMON_DIR)/procfs.c
+cgroup.o: $(DAEMONS_COMMON_DIR)/cgroup.c
+ $(CC) $(CFLAGS) $(MEMCFLAG) -c $(DAEMONS_COMMON_DIR)/cgroup.c
+
binding_support.o: $(DAEMONS_COMMON_DIR)/binding_support.c
$(CC) $(CFLAGS) $(MEMCFLAG) -c
$(DAEMONS_COMMON_DIR)/binding_support.c
diff -ruN sge-8.1.8-orig/source/daemons/common/cgroup.c
sge-8.1.8/source/daemons/common/cgroup.c
--- sge-8.1.8-orig/source/daemons/common/cgroup.c 1969-12-31
19:00:00.000000000 -0500
+++ sge-8.1.8/source/daemons/common/cgroup.c 2015-03-31 14:18:00.012710650
-0400
@@ -0,0 +1,342 @@
+/* cgroup.c -- generic cgroup routines
+ Copyright (C) 2012 Mark Dixon, University of Leeds <m.c.dixon at
leeds.ac.uk>
+
+ This file is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License
+ as published by the Free Software Foundation; either version 3 of
+ the License, or (at your option) any later version.
+
+ This file is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this file. If not, a copy can be downloaded
+ from http://www.gnu.org/licenses/lgpl.html.
+*/
+
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "uti/sge_rmon.h"
+#include "uti/sge_log.h"
+#include "uti/sge_unistd.h"
+#include "uti/sge_stdio.h"
+#include "uti/sge_dstring.h"
+#include "uti/sge_parse_num_par.h"
+
+#include "sgeobj/sge_conf.h"
+
+#include "basis_types.h"
+#include "cgroup.h"
+
+bool set_cgroup_value(const dstring *cgroup, const char *attrib, sge_rlim_t
value);
+
+/****** cgroup/create_cgroup_memory() ****************************************
+*
+* NAME
+* create_cgroup_memory -- create a memory cgroup on current host
+*
+* SYNOPSIS
+* int create_cgroup_memory(u_long32 job_id, u_long32 ja_task_id,
+* const char *queue, sge_rlim_t rss, sge_rlim_t vmem)
+*
+* FUNCTION
+* Creates a cgroup for the specified job on the current execution host.
+*
+* It *should* create the cgroup only for the required queue instance,
+* permitting jobs to straddle multiple queues on the same host. This is not
+* done as I couldn't figure out an easy way to get the queue names in the
+* PDC. Perhaps the PDC should glob for all cgroups that match the job.
+*
+* INPUTS
+* job_id - job id
+* ja_task_id - task id
+* queue - name of queue (currently unused)
+* rss - RAM required ("memory.limit_in_bytes")
+* vmem - RAM/Swap required ("memory.memsw.limit_in_bytes")
+*
+* RESULT
+* 0 on success
+* -2 if there is a problem with the host
+* -3 if there is a problem with the job
+******************************************************************************/
+
+int create_cgroup_memory(u_long32 job_id, u_long32 ja_task_id,
+const char *queue, sge_rlim_t rss, sge_rlim_t vmem) {
+DENTER(TOP_LAYER, "create_cgroup_memory");
+
+DPRINTF(("create cgroup: job_id = "sge_u32" ja_task_id = "sge_u32" rss =
"sge_u64" vmem = "sge_u64"\n", job_id, ja_task_id, rss, vmem));
+
+ /* Sanity-check arguments */
+
+if (rss == 0 && vmem < 0) {
+ /* Not being asked to do anything */
+DRETURN(0);
+} else if (rss == 0 && vmem > 0) {
+ /* kernel doesn't allow vmem to be set unless rss is set */
+INFO((SGE_EVENT, "cgroup memory: vmem set but rss isn't"));
+DRETURN(-3);
+}
+
+ /* Get name of cgroup to create */
+dstring cgroup = get_cgroup_memory_name(job_id, ja_task_id);
+
+ /* Create memory cgroup */
+if (sge_mkdir(sge_dstring_get_string(&cgroup), 0755, false, false) != 0) {
+INFO((SGE_EVENT, "cgroup memory: failed to create cgroup."));
+DRETURN(-2);
+}
+
+ /* Assign resources */
+
+ int retval = 0;
+
+ /* If this goes wrong, probably a cgroups/host problem */
+ if (retval == 0 && rss > 0 && ! set_cgroup_value(&cgroup,
"memory.limit_in_bytes", rss)) {
+ INFO((SGE_EVENT, "cgroup memory: Couldn't set vmem limit."));
+ retval = -2;
+ }
+
+ /* If this goes wrong, could be because rss < vmem */
+ if (retval == 0 && vmem > 0 && ! set_cgroup_value(&cgroup,
"memory.memsw.limit_in_bytes", vmem)) {
+ INFO((SGE_EVENT, "cgroup memory: Couldn't set memsw limit."));
+ retval = -3;
+ }
+
+ /* Tidy cgroup away if we've hit an error condition */
+ if (retval != 0) {
+ remove_cgroup(sge_dstring_get_string(&cgroup));
+ }
+
+ sge_dstring_free(&cgroup);
+
+ DRETURN(retval);
+}
+
+/****** cgroup/get_cgroup_memory_name()
****************************************
+*
+* NAME
+* cgroup/get_cgroup_memory_name -- generate a name for a memory cgroup
+*
+* SYNOPSIS
+* dstring get_cgroup_memory_name(u_long32 job_id, u_long32 ja_task_id)
+*
+* FUNCTION
+* Creates a unique name for a memory cgroup for an exec host to use.
+* It *should* insert the queue name (see create_cgroup_memory discussion)
+* but doesn't.
+*
+* INPUTS
+* job_id - job id
+* ja_task_id - task id
+*
+* RESULT
+* dstring containing name on success (CALLER MUST FREE RETURNED dstring)
+* NULL if memory cgroup isn't activiated in config
+******************************************************************************/
+
+
+dstring get_cgroup_memory_name(u_long32 job_id, u_long32 ja_task_id) {
+ dstring name = DSTRING_INIT;
+
+ char *cgroup_memory = mconf_get_cgroup_memory();
+ if (cgroup_memory != NULL) {
+ sge_dstring_sprintf(&name, "%s/"sge_u32"."sge_u32, cgroup_memory,
job_id, ja_task_id);
+ sge_free(&cgroup_memory);
+ }
+
+ return name;
+}
+
+/****** cgroup/set_cgroup_value() ****************************************
+*
+* NAME
+* cgroup/set_cgroup_value -- set a cgroup attribute
+*
+* SYNOPSIS
+* bool set_cgroup_value(const dstring *cgroup, const char *attrib,
+* sge_rlim_t value)
+*
+* FUNCTION
+* Modify a cgroup attribute.
+*
+* INPUTS
+* cgroup - filename of cgroup
+* attrib - attribute to modify
+* value - value to set attribute to
+*
+* RESULT
+* true on success / false on failure
+******************************************************************************/
+
+bool set_cgroup_value(const dstring *cgroup, const char *attrib, sge_rlim_t
value) {
+ bool retval = false;
+
+ DENTER(TOP_LAYER, "set_cgroup_value");
+
+ if (value == 0) {
+ return false;
+ }
+
+ dstring file = DSTRING_INIT;
+ sge_dstring_copy_dstring(&file, cgroup);
+ sge_dstring_append(&file, "/");
+ sge_dstring_append(&file, attrib);
+
+ FILE *fp;
+ FOPEN(fp, sge_dstring_get_string(&file), "w");
+ FPRINTF((fp, sge_u64, value));
+ FCLOSE(fp);
+
+ retval = true;
+
+FOPEN_ERROR:
+FPRINTF_ERROR:
+FCLOSE_ERROR:
+ sge_dstring_free(&file);
+
+ DRETURN(retval);
+}
+
+/****** cgroup/get_cgroup_double_value() ************************************
+*
+* NAME
+* cgroup/get_cgroup_double_value -- return a cgroup attribute
+*
+* SYNOPSIS
+* bool get_cgroup_double_value(const dstring *cgroup, const char *attrib,
+* double *value)
+*
+* FUNCTION
+* Return a cgroup attribute, where the value is expected to be compatible
+* with type double.
+*
+* INPUTS
+* cgroup - filename of cgroup
+* attrib - attribute to return
+* value - returned value
+*
+* RESULT
+* true on success / false on failure
+******************************************************************************/
+
+bool get_cgroup_double_value(const dstring *cgroup, const char *attrib, double
*value) {
+ bool retval = false;
+
+ DENTER(TOP_LAYER, "get_cgroup_ulong64_value");
+
+ dstring file = DSTRING_INIT;
+ sge_dstring_copy_dstring(&file, cgroup);
+ sge_dstring_append(&file, "/");
+ sge_dstring_append(&file, attrib);
+
+ FILE *fp;
+ FOPEN(fp, sge_dstring_get_string(&file), "r");
+ FSCANF((fp, "%le", value));
+ FCLOSE(fp);
+
+ retval = true;
+
+FOPEN_ERROR:
+FSCANF_ERROR:
+FCLOSE_ERROR:
+ sge_dstring_free(&file);
+
+ DRETURN(retval);
+}
+
+/****** cgroup/join_cgroup() ************************************************
+*
+* NAME
+* cgroup/join_cgroup -- add current process to cgroup
+*
+* SYNOPSIS
+* bool join_cgroup(char *cgroup)
+*
+* FUNCTION
+* Add the this process to the specified cgroup.
+*
+* INPUTS
+* cgroup - filename of cgroup
+*
+* RESULT
+* true on success / false on failure
+******************************************************************************/
+
+bool join_cgroup(char *cgroup) {
+ bool retval = false;
+
+ dstring tasks = DSTRING_INIT;
+ sge_dstring_copy_string(&tasks, cgroup);
+ sge_dstring_append(&tasks, "/tasks");
+
+ /* Seem to need euid == uid, otherwise don't have permission */
+ /* to move ourselves into a new cgroup */
+ uid_t uid = getuid();
+ uid_t euid = geteuid();
+ if (uid != euid) {
+ sge_seteuid(uid);
+ }
+
+ /* Add current PID to cgroup's tasks */
+ FILE *fp;
+ FOPEN(fp, sge_dstring_get_string(&tasks), "w");
+ FPRINTF((fp, uid_t_fmt, getpid()));
+ FCLOSE(fp);
+
+ retval = true;
+
+FOPEN_ERROR:
+FPRINTF_ERROR:
+FCLOSE_ERROR:
+
+ /* Return to original euid. */
+ if (euid != geteuid()) {
+ sge_seteuid(euid);
+ }
+
+ sge_dstring_free(&tasks);
+
+ return retval;
+}
+
+/****** cgroup/remove_cgroup() ***********************************************
+*
+* NAME
+* cgroup/remove_cgroup -- delete the specified cgroup
+*
+* SYNOPSIS
+* bool remove_cgroup(const char *cgroup)
+*
+* FUNCTION
+* Removes the specified cgroup.
+*
+* INPUTS
+* cgroup - filename of cgroup
+*
+* RESULT
+* true on success / false on failure
+******************************************************************************/
+
+bool remove_cgroup(const char *cgroup) {
+ bool retval = true;
+
+ DENTER(TOP_LAYER, "remove_cgroup");
+
+ if (cgroup != NULL && strncmp(cgroup, "NONE", strlen(cgroup)) != 0) {
+ /*
+ * Don't use sge_rmdir - it halts early because it cannot get rid
+ * of "files" like memory.memsw.limit_in_bytes, etc.
+ */
+ if (rmdir(cgroup)) {
+ DPRINTF(("could not rmdir cgroup %s\n", cgroup));
+ retval = false;
+ } else {
+ DPRINTF(("removed cgroup %s\n", cgroup));
+ }
+ }
+
+ DRETURN(retval);
+}
diff -ruN sge-8.1.8-orig/source/daemons/common/cgroup.h
sge-8.1.8/source/daemons/common/cgroup.h
--- sge-8.1.8-orig/source/daemons/common/cgroup.h 1969-12-31
19:00:00.000000000 -0500
+++ sge-8.1.8/source/daemons/common/cgroup.h 2015-03-31 14:18:00.026568000
-0400
@@ -0,0 +1,29 @@
+/* cgroup.h -- generic cgroup routines
+ Copyright (C) 2012 Mark Dixon, University of Leeds <m.c.dixon at
leeds.ac.uk>
+
+ This file is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License
+ as published by the Free Software Foundation; either version 3 of
+ the License, or (at your option) any later version.
+
+ This file is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this file. If not, a copy can be downloaded
+ from http://www.gnu.org/licenses/lgpl.html.
+*/
+
+
+#include "uti/sge_dstring.h"
+#include "uti/sge_parse_num_par.h"
+
+/* creates cgroup container for job */
+int create_cgroup_memory(u_long32 job_id, u_long32 ja_task_id, const char
*queue, sge_rlim_t rss, sge_rlim_t vmem);
+bool join_cgroup(char *cgroup);
+dstring get_cgroup_memory_name(u_long32 job_id, u_long32 ja_task_id);
+bool remove_cgroup(const char *cgroup);
+
+bool get_cgroup_double_value(const dstring *cgroup, const char *attrib, double
*value);
diff -ruN sge-8.1.8-orig/source/daemons/execd/Makefile
sge-8.1.8/source/daemons/execd/Makefile
--- sge-8.1.8-orig/source/daemons/execd/Makefile 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/execd/Makefile 2015-03-31 14:18:00.054507000
-0400
@@ -69,7 +69,8 @@
sig_handlers.c \
startprog.c \
shutdown.c \
- sge_mt_init.c
+ sge_mt_init.c \
+ cgroup.c
EXECD_OBJS = $(EXECD_SRC:.c=.o)
diff -ruN sge-8.1.8-orig/source/daemons/execd/exec_job.c
sge-8.1.8/source/daemons/execd/exec_job.c
--- sge-8.1.8-orig/source/daemons/execd/exec_job.c 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/execd/exec_job.c 2015-03-31 14:18:00.044165321
-0400
@@ -106,6 +106,7 @@
#include "msg_common.h"
#include "msg_execd.h"
#include "msg_daemons_common.h"
+#include "cgroup.h"
#include "binding_support.h"
#if defined(SOLARIS)
@@ -321,7 +322,7 @@
bool bInputFileStaging, bOutputFileStaging, bErrorFileStaging;
const char* processor_binding_strategy = NULL;
-
+ char *cgroup_memory;
/* env var reflecting SGE_BINDING if set */
char* sge_binding_environment = NULL;
/* string reflecting the logical socket,core pairs if "pe" is set */
@@ -482,6 +483,47 @@
INFO((SGE_EVENT, "appended socket,core list to hostfile %s",
rankfileinput));
}
+
+ /***************** cgroup part
******************************************/
+ cgroup_memory = mconf_get_cgroup_memory();
+ if (cgroup_memory) {
+ sge_free(&cgroup_memory);
+
+ sge_rlim_t cgroup_rss = 0;
+ sge_rlim_t cgroup_vmem = 0;
+
+ if (strncmp(lGetString(master_q, QU_h_vmem), "INFINITY", 8) != 0) {
+ /* TODO: no error checking. Supposed to use */
+ /* extended_parse_ulong_val but it's only 32-bit. */
+ const char *vmem = lGetString(master_q, QU_h_vmem);
+ sge_parse_num_val(&cgroup_rss, NULL, vmem, vmem, NULL, 0);
+ sge_parse_num_val(&cgroup_vmem, NULL, vmem, vmem, NULL, 0);
+ }
+
+ if (strncmp(lGetString(master_q, QU_h_rss), "INFINITY", 8) != 0) {
+ /* TODO: no error checking. Supposed to use
extended_parse_ulong_val */
+ /* but it's only 32-bit.
*/
+ const char *vmem = lGetString(master_q, QU_h_rss);
+ sge_parse_num_val(&cgroup_rss, NULL, vmem, vmem, NULL, 0);
+ }
+
+ /* Scale memory based on number of slots */
+ cgroup_rss *= (sge_rlim_t) lGetUlong(master_q, QU_job_slots);
+ cgroup_vmem *= (sge_rlim_t) lGetUlong(master_q, QU_job_slots);
+
+ int ret = create_cgroup_memory(job_id, ja_task_id,
+ lGetString(master_q, QU_qname),
+ cgroup_rss, cgroup_vmem);
+
+ if (ret == 0) {
+ INFO((SGE_EVENT, "created memory cgroup"));
+ } else {
+ snprintf(err_str, err_length, SFNMAX, MSG_CGROUP_CREATEFAIL);
+ DRETURN(ret);
+ }
+ }
+
+
/***************** write out sge host file
******************************/
/* JG: TODO: create function write_pe_hostfile() */
/* JG: TODO (254) use function sge_get_active_job.... */
@@ -1165,6 +1207,9 @@
fprintf(fp, "h_fsize=%s\n", lGetString(master_q, QU_h_fsize));
fprintf(fp, "s_fsize=%s\n", lGetString(master_q, QU_s_fsize));
+ fprintf(fp, "h_as=%s\n", lGetString(master_q, QU_h_as));
+ fprintf(fp, "s_as=%s\n", lGetString(master_q, QU_s_as));
+
{
char *s;
@@ -1335,6 +1380,16 @@
fprintf(fp, "binding=%s\n", processor_binding_strategy);
}
+ {
+ /* memory cgroup information */
+ dstring cgroup = get_cgroup_memory_name(job_id, ja_task_id);
+ if (sge_dstring_get_string(&cgroup) == NULL) {
+ fprintf(fp, "cgroup_memory=NONE\n");
+ } else {
+ fprintf(fp, "cgroup_memory=%s\n", sge_dstring_get_string(&cgroup));
+ }
+ sge_dstring_free(&cgroup);
+ }
if(petep != NULL) {
fprintf(fp, "job_name=%s\n", lGetString(petep, PET_name));
} else {
diff -ruN sge-8.1.8-orig/source/daemons/execd/msg_execd.h
sge-8.1.8/source/daemons/execd/msg_execd.h
--- sge-8.1.8-orig/source/daemons/execd/msg_execd.h 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/execd/msg_execd.h 2015-03-31 14:18:00.061899516
-0400
@@ -288,5 +288,10 @@
#define MSG_EXECD_DISABLEDELEAYDJOBREPORTING _MESSAGE(29191, _("Delayed
job reporting period finished"))
#define MSG_EXECD_ENABLEDELEAYDREPORTINGFORJOB_U _MESSAGE(29192, _("Enable
delayed job reporting for job "sge_U32CFormat""))
+/*
+** Misc (waiting for renumbering)
+*/
+#define MSG_CGROUP_CREATEFAIL _MESSAGE(29193, _("cannot create
cgroup"))
+
#endif /* __MSG_EXECD_H */
diff -ruN sge-8.1.8-orig/source/daemons/execd/ptf.c
sge-8.1.8/source/daemons/execd/ptf.c
--- sge-8.1.8-orig/source/daemons/execd/ptf.c 2014-10-23 18:03:56.000000000
-0400
+++ sge-8.1.8/source/daemons/execd/ptf.c 2015-03-31 14:18:00.068324069
-0400
@@ -100,6 +100,7 @@
#include "sgedefs.h"
#include "exec_ifm.h"
#include "pdc.h"
+#include "cgroup.h"
/*
*
@@ -908,14 +909,49 @@
jobs->jd_rwtime_c + jobs->jd_rwtime_a);
}
+ double vmem = 0;
+ double himem = 0;
+ dstring cgroup_memory = get_cgroup_memory_name(lGetUlong(job,
JL_job_ID), lGetUlong(osjob, JO_ja_task_ID));
+ if (sge_dstring_get_string(&cgroup_memory) != NULL) {
+ /* Retrieve usage from cgroups */
+
+ /* TODO: really ought to check get_cgroup_double_value */
+ /* return codes */
+
+ double failcnt = 0;
+ double vmem_lim = 0;
+
+ /* Retrieve vmem cgroup value */
+ get_cgroup_double_value(&cgroup_memory,
"memory.memsw.usage_in_bytes", &vmem);
+
+ /* Retrieve himem cgroup value */
+ get_cgroup_double_value(&cgroup_memory,
"memory.memsw.max_usage_in_bytes", &himem);
+
+ /* If failcnt > 0, we have hit the limit so fabricate an entry
*/
+ /* that's too big */
+ get_cgroup_double_value(&cgroup_memory, "memory.memsw.failcnt",
&failcnt);
+ if (failcnt > 0) {
+ get_cgroup_double_value(&cgroup_memory,
"memory.memsw.limit_in_bytes", &vmem_lim);
+ vmem = vmem_lim +1;
+ himem = vmem_lim +1;
+ }
+ } else {
+ /* Retrieve usage from ptf */
+
+ vmem = jobs->jd_vmem;
+ himem = jobs->jd_himem;
+ }
+ sge_dstring_free(&cgroup_memory);
+
+
/* set vmem usage */
if ((usage = lGetElemStr(usage_list, UA_name, USAGE_ATTR_VMEM))) {
- lSetDouble(usage, UA_value, jobs->jd_vmem);
+ lSetDouble(usage, UA_value, vmem);
}
/* set himem usage */
if ((usage = lGetElemStr(usage_list, UA_name,
USAGE_ATTR_MAXVMEM))) {
- lSetDouble(usage, UA_value, jobs->jd_himem);
+ lSetDouble(usage, UA_value, himem);
}
/* build new pid list */
@@ -958,7 +994,7 @@
tmp_jobs->jd_utime_c + tmp_jobs->jd_utime_a +
tmp_jobs->jd_stime_c + tmp_jobs->jd_stime_a,
tmp_jobs->jd_mem, tmp_jobs->jd_chars,
- (double) tmp_jobs->jd_vmem, (double) tmp_jobs->jd_himem));
+ vmem, himem));
} else {
/*
* NOTE: Under what conditions would DC have a job
diff -ruN sge-8.1.8-orig/source/daemons/execd/reaper_execd.c
sge-8.1.8/source/daemons/execd/reaper_execd.c
--- sge-8.1.8-orig/source/daemons/execd/reaper_execd.c 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/execd/reaper_execd.c 2015-03-31
14:18:00.076627680 -0400
@@ -101,6 +101,7 @@
#include "msg_common.h"
#include "msg_daemons_common.h"
#include "msg_execd.h"
+#include "cgroup.h"
#include "binding_support.h"
#ifdef COMPILE_DC
@@ -488,6 +489,11 @@
clean_up_binding(binding);
+ /*
+ * cgroups
+ */
+ remove_cgroup(get_conf_val("cgroup_memory"));
+
/*
* look for exit status of shepherd This is the last file the shepherd
* creates. So if we can find this shepherd terminated normal.
diff -ruN
sge-8.1.8-orig/source/daemons/qmaster/sge_advance_reservation_qmaster.c
sge-8.1.8/source/daemons/qmaster/sge_advance_reservation_qmaster.c
--- sge-8.1.8-orig/source/daemons/qmaster/sge_advance_reservation_qmaster.c
2014-10-23 18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/qmaster/sge_advance_reservation_qmaster.c
2015-03-31 14:18:00.087717616 -0400
@@ -1632,12 +1632,15 @@
QU_h_rss,
QU_s_vmem,
QU_h_vmem,
+ QU_s_as,
+ QU_h_as,
NoName };
static char *value = "INFINITY";
static int attr[] = {
QU_s_cpu, QU_h_cpu, QU_s_fsize, QU_h_fsize, QU_s_data,
QU_h_data, QU_s_stack, QU_h_stack, QU_s_core, QU_h_core,
- QU_s_rss, QU_h_rss, QU_s_vmem, QU_h_vmem, NoName
+ QU_s_rss, QU_h_rss, QU_s_vmem, QU_h_vmem, QU_s_as, QU_h_as,
+ NoName
};
lDescr *rdp = NULL;
diff -ruN sge-8.1.8-orig/source/daemons/qmaster/sge_give_jobs.c
sge-8.1.8/source/daemons/qmaster/sge_give_jobs.c
--- sge-8.1.8-orig/source/daemons/qmaster/sge_give_jobs.c 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/qmaster/sge_give_jobs.c 2015-03-31
14:18:00.096173568 -0400
@@ -180,6 +180,8 @@
QU_h_rss,
QU_s_vmem,
QU_h_vmem,
+ QU_s_as,
+ QU_h_as,
NoName };
@@ -334,6 +336,8 @@
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_fsize,
"h_fsize");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_s_vmem,
"s_vmem");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_vmem,
"h_vmem");
+ reduce_queue_limit(master_centry_list, qep, tmpjep, QU_s_as, "s_as");
+ reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_as, "h_as");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_s_rt, "s_rt");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_rt, "h_rt");
@@ -603,6 +607,8 @@
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_fsize,
"h_fsize");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_s_vmem,
"s_vmem");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_vmem,
"h_vmem");
+ reduce_queue_limit(master_centry_list, qep, tmpjep, QU_s_as, "s_as");
+ reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_as, "h_as");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_s_rt, "s_rt");
reduce_queue_limit(master_centry_list, qep, tmpjep, QU_h_rt, "h_rt");
diff -ruN sge-8.1.8-orig/source/daemons/qmaster/sge_qinstance_qmaster.c
sge-8.1.8/source/daemons/qmaster/sge_qinstance_qmaster.c
--- sge-8.1.8-orig/source/daemons/qmaster/sge_qinstance_qmaster.c
2014-10-23 18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/qmaster/sge_qinstance_qmaster.c 2015-03-31
14:18:00.102632016 -0400
@@ -205,6 +205,8 @@
case CQ_h_rss:
case CQ_s_vmem:
case CQ_h_vmem:
+ case CQ_s_as:
+ case CQ_h_as:
{
const char *old_value = lGetString(this_elem, attribute_name);
const char *new_value = NULL;
diff -ruN sge-8.1.8-orig/source/daemons/qmaster/sge_sched_prepare_data.c
sge-8.1.8/source/daemons/qmaster/sge_sched_prepare_data.c
--- sge-8.1.8-orig/source/daemons/qmaster/sge_sched_prepare_data.c
2014-10-23 18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/qmaster/sge_sched_prepare_data.c 2015-03-31
14:18:00.109767580 -0400
@@ -93,6 +93,7 @@
CQ_h_rt,
CQ_h_stack,
CQ_h_vmem,
+ CQ_h_as,
CQ_min_cpu_interval,
CQ_rerun,
CQ_s_core,
@@ -103,6 +104,7 @@
CQ_s_rss,
CQ_s_stack,
CQ_s_vmem,
+ CQ_s_as,
NoName
};
@@ -149,6 +151,8 @@
QU_h_rss,
QU_s_vmem,
QU_h_vmem,
+ QU_s_as,
+ QU_h_as,
QU_min_cpu_interval,
QU_suspended_on_subordinate,
diff -ruN sge-8.1.8-orig/source/daemons/shepherd/Makefile
sge-8.1.8/source/daemons/shepherd/Makefile
--- sge-8.1.8-orig/source/daemons/shepherd/Makefile 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/shepherd/Makefile 2015-03-31 14:18:00.115930000
-0400
@@ -45,7 +45,8 @@
sge_pset.c \
sge_fileio.c \
sge_shepherd_ijs.c \
- shepherd_binding.c
+ shepherd_binding.c \
+ shepherd_cgroup.c
SHEPHERD_ADD_SRC = \
config_file.c \
@@ -59,6 +60,7 @@
sge_mt_init.c \
sge_processes_irix.c \
sge_spool.c \
+ cgroup.c \
binding_support.c
SHEPHERD_OBJS = $(SHEPHERD_SRC:.c=.o) sge_ijs_comm.o sge_ijs_threads.o
@@ -133,3 +135,6 @@
shepherd_binding.o: $(SHEPHERD_DIR)/shepherd_binding.c
$(CC) -I$(SHEPHERD_DIR) $(CFLAGS) -c
$(SHEPHERD_DIR)/shepherd_binding.c
+shepherd_cgroup.o: $(SHEPHERD_DIR)/shepherd_cgroup.c
+ $(CC) -I$(SHEPHERD_DIR) $(CFLAGS) -c
$(SHEPHERD_DIR)/shepherd_cgroup.c
+
diff -ruN sge-8.1.8-orig/source/daemons/shepherd/setrlimits.c
sge-8.1.8/source/daemons/shepherd/setrlimits.c
--- sge-8.1.8-orig/source/daemons/shepherd/setrlimits.c 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/shepherd/setrlimits.c 2015-03-31
14:18:00.122140158 -0400
@@ -131,6 +131,9 @@
sge_rlim_t s_vmem, s_vmem_is_consumable_job;
sge_rlim_t h_vmem, h_vmem_is_consumable_job;
+ sge_rlim_t s_as;
+ sge_rlim_t h_as;
+
sge_rlim_t s_fsize;
sge_rlim_t h_fsize;
@@ -199,6 +202,9 @@
PARSE_IT(&s_rss, "s_rss");
PARSE_IT(&h_rss, "h_rss");
+ PARSE_IT(&s_as, "s_as");
+ PARSE_IT(&h_as, "h_as");
+
PARSE_IT(&s_fsize, "s_fsize");
PARSE_IT(&h_fsize, "h_fsize");
@@ -223,6 +229,8 @@
h_data = RL_MAX(h_data, LIMIT_VMEM_MIN);
s_rss = RL_MAX(s_rss, LIMIT_VMEM_MIN);
h_rss = RL_MAX(h_rss, LIMIT_VMEM_MIN);
+ s_as = RL_MAX(s_as, LIMIT_VMEM_MIN);
+ h_as = RL_MAX(h_as, LIMIT_VMEM_MIN);
s_stack = RL_MAX(s_stack, LIMIT_STACK_MIN);
h_stack = RL_MAX(h_stack, LIMIT_STACK_MIN);
s_cpu = RL_MAX(s_cpu, LIMIT_CPU_MIN);
@@ -266,6 +274,9 @@
if (s_vmem > h_vmem) {
s_vmem = h_vmem;
}
+ if (s_as > h_as) {
+ s_as = h_as;
+ }
/*s_data = RL_MIN(s_data, s_vmem);
s_stack = RL_MIN(s_stack, s_vmem);
h_data = RL_MIN(h_data, h_vmem);
@@ -403,12 +414,24 @@
# endif
# if defined(RLIMIT_VMEM)
- rlp.rlim_cur = s_vmem;
- rlp.rlim_max = h_vmem;
+ char *cgroup_memory = get_conf_val("cgroup_memory");
+ if (strncmp(cgroup_memory, "NONE", strlen(cgroup_memory)) == 0) {
+ rlp.rlim_cur = s_vmem;
+ rlp.rlim_max = h_vmem;
+ } else {
+ rlp.rlim_cur = s_as;
+ rlp.rlim_max = h_as;
+ }
pushlimit(RLIMIT_VMEM, &rlp, trace_rlimit);
# elif defined(RLIMIT_AS)
- rlp.rlim_cur = s_vmem;
- rlp.rlim_max = h_vmem;
+ char *cgroup_memory = get_conf_val("cgroup_memory");
+ if (strncmp(cgroup_memory, "NONE", strlen(cgroup_memory)) == 0) {
+ rlp.rlim_cur = s_vmem;
+ rlp.rlim_max = h_vmem;
+ } else {
+ rlp.rlim_cur = s_as;
+ rlp.rlim_max = h_as;
+ }
pushlimit(RLIMIT_AS, &rlp, trace_rlimit);
# endif
diff -ruN sge-8.1.8-orig/source/daemons/shepherd/shepherd.c
sge-8.1.8/source/daemons/shepherd/shepherd.c
--- sge-8.1.8-orig/source/daemons/shepherd/shepherd.c 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/shepherd/shepherd.c 2015-03-31
14:18:00.129195713 -0400
@@ -50,6 +50,7 @@
#include "uti/sge_binding_hlp.h"
#include "uti/sge_string.h"
#include "shepherd_binding.h"
+#include "shepherd_cgroup.h"
/* TODO: (SH) get_path.h is a header-file of execd. We have to do a CLEANUP
here. */
#include "get_path.h"
@@ -894,6 +895,13 @@
cpusetting();
+ /*
+ * Deal with cgroups
+ */
+ if (! do_cgroups()) {
+ shepherd_error(1, "cgroup processing failed");
+ }
+
/*
* this blocks sge_shepherd until the first time the token is
* successfully set, then we start the sge_coshepherd in background
diff -ruN sge-8.1.8-orig/source/daemons/shepherd/shepherd_cgroup.c
sge-8.1.8/source/daemons/shepherd/shepherd_cgroup.c
--- sge-8.1.8-orig/source/daemons/shepherd/shepherd_cgroup.c 1969-12-31
19:00:00.000000000 -0500
+++ sge-8.1.8/source/daemons/shepherd/shepherd_cgroup.c 2015-03-31
14:18:00.140277000 -0400
@@ -0,0 +1,63 @@
+/* shepherd_cgroup.c -- cgroup routines for shepherd
+ Copyright (C) 2012 Mark Dixon, University of Leeds <m.c.dixon at
leeds.ac.uk>
+
+ This file is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License
+ as published by the Free Software Foundation; either version 3 of
+ the License, or (at your option) any later version.
+
+ This file is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this file. If not, a copy can be downloaded
+ from http://www.gnu.org/licenses/lgpl.html.
+*/
+
+#include <string.h>
+
+#include "uti/config_file.h"
+#include "uti/sge_unistd.h"
+
+#include "err_trace.h"
+#include "cgroup.h"
+
+/****** shepherd_cgroup/do_cgroups()
***********************************************
+*
+* NAME
+* shepherd_cgroup/do_cgroups -- join shepherd to relevant cgroups
+*
+* SYNOPSIS
+* bool do_cgroups(void)
+*
+* FUNCTION
+* Join the job's cgroups (if enabled)
+*
+* INPUTS
+*
+* RESULT
+* true on success / false on failure
+******************************************************************************/
+bool do_cgroups(void) {
+ char *cgroup_memory = get_conf_val("cgroup_memory");
+
+ if (cgroup_memory == NULL) {
+ shepherd_trace("do_cgroups: \"cgroup_memory\" parameter not found in
config file");
+ return false;
+ }
+
+ /* Early exit if we don't need to do anything */
+ if (strncmp(cgroup_memory, "NONE", strlen(cgroup_memory)) == 0) {
+ return true;
+ }
+
+ if (! join_cgroup(cgroup_memory)) {
+ shepherd_trace("do_cgroups: failed to join cgroup");
+ return false;
+ }
+
+ shepherd_trace("do_cgroups: joined memory cgroup %s", cgroup_memory);
+ return true;
+}
diff -ruN sge-8.1.8-orig/source/daemons/shepherd/shepherd_cgroup.h
sge-8.1.8/source/daemons/shepherd/shepherd_cgroup.h
--- sge-8.1.8-orig/source/daemons/shepherd/shepherd_cgroup.h 1969-12-31
19:00:00.000000000 -0500
+++ sge-8.1.8/source/daemons/shepherd/shepherd_cgroup.h 2015-03-31
14:18:00.167191000 -0400
@@ -0,0 +1,19 @@
+/* shepherd_cgroup.h -- cgroup routines for shepherd
+ Copyright (C) 2012 Mark Dixon, University of Leeds <m.c.dixon at
leeds.ac.uk>
+
+ This file is free software: you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public License
+ as published by the Free Software Foundation; either version 3 of
+ the License, or (at your option) any later version.
+
+ This file is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with this file. If not, a copy can be downloaded
+ from http://www.gnu.org/licenses/lgpl.html.
+*/
+
+bool do_cgroups(void);
diff -ruN sge-8.1.8-orig/source/libs/sgeobj/sge_centry.c
sge-8.1.8/source/libs/sgeobj/sge_centry.c
--- sge-8.1.8-orig/source/libs/sgeobj/sge_centry.c 2014-10-23
18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_centry.c 2015-03-31 14:18:00.169898994
-0400
@@ -101,7 +101,7 @@
{"m_topology_inuse",0,0, 0, TYPE_STR}
};
-const int max_queue_resources=24; /* specifies the number of elements in the
queue_resource array */
+const int max_queue_resources=26; /* specifies the number of elements in the
queue_resource array */
const struct queue2cmplx queue_resource[] = {
{"qname", QU_qname, 0, 0,
TYPE_STR },
{"hostname", QU_qhostname, 0, 0,
TYPE_HOST},
@@ -126,6 +126,8 @@
{"h_rss", QU_h_rss, CQ_h_rss, AMEM_value,
TYPE_MEM }, /* value is SGE_STRING */
{"s_vmem", QU_s_vmem, CQ_s_vmem, AMEM_value,
TYPE_MEM }, /* value is SGE_STRING */
{"h_vmem", QU_h_vmem, CQ_h_vmem, AMEM_value,
TYPE_MEM }, /* value is SGE_STRING */
+ {"s_as", QU_s_as, CQ_s_as, AMEM_value,
TYPE_MEM }, /* value is SGE_STRING */
+ {"h_as", QU_h_as, CQ_h_as, AMEM_value,
TYPE_MEM }, /* value is SGE_STRING */
{"min_cpu_interval", QU_min_cpu_interval, CQ_min_cpu_interval,
AINTER_value, TYPE_TIM } /* value is SGE_STRING */
};
diff -ruN sge-8.1.8-orig/source/libs/sgeobj/sge_conf.c
sge-8.1.8/source/libs/sgeobj/sge_conf.c
--- sge-8.1.8-orig/source/libs/sgeobj/sge_conf.c 2014-10-23
18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_conf.c 2015-03-31 14:18:00.176952557
-0400
@@ -190,6 +190,7 @@
static bool keep_active = false;
static bool enable_windomacc = false;
static bool enable_binding = false;
+static char* cgroup_memory = NULL;
static bool enable_addgrp_kill = true;
static u_long32 pdc_interval = 1;
static char s_descriptors[100];
@@ -874,6 +875,7 @@
keep_active = false;
enable_windomacc = false;
enable_binding = HAVE_HWLOC ? true : false;
+ sge_free(&cgroup_memory);
enable_addgrp_kill = true;
use_qsub_gid = false;
prof_execd_thrd = false;
@@ -933,6 +935,10 @@
if (parse_bool_param(s, "ENABLE_BINDING", &enable_binding)) {
continue;
}
+ if (!strncasecmp(s, "CGROUP_MEMORY", sizeof("CGROUP_MEMORY")-1)) {
+ cgroup_memory = sge_strdup(NULL, &(s[sizeof("CGROUP_MEMORY")]));
+ continue;
+ }
if (parse_bool_param(s, "ENABLE_ADDGRP_KILL", &enable_addgrp_kill)) {
continue;
}
@@ -2098,6 +2104,20 @@
DRETURN(ret);
}
+/* returned pointer needs to be freed */
+char* mconf_get_cgroup_memory(void) {
+ char* ret = NULL;
+
+ DENTER(BASIS_LAYER, "mconf_get_cgroup_memory");
+ SGE_LOCK(LOCK_MASTER_CONF, LOCK_READ);
+
+ ret = sge_strdup(ret, cgroup_memory);
+
+ SGE_UNLOCK(LOCK_MASTER_CONF, LOCK_READ);
+ DRETURN(ret);
+}
+
+
bool mconf_get_enable_addgrp_kill(void) {
bool ret;
diff -ruN sge-8.1.8-orig/source/libs/sgeobj/sge_conf.h
sge-8.1.8/source/libs/sgeobj/sge_conf.h
--- sge-8.1.8-orig/source/libs/sgeobj/sge_conf.h 2014-10-23
18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_conf.h 2015-03-31 14:18:00.186024000
-0400
@@ -117,6 +117,7 @@
bool mconf_get_keep_active(void);
bool mconf_get_enable_windomacc(void);
bool mconf_get_enable_binding(void);
+char* mconf_get_cgroup_memory(void);
bool mconf_get_simulate_execds(void);
bool mconf_get_simulate_jobs(void);
long mconf_get_ptf_max_priority(void);
diff -ruN sge-8.1.8-orig/source/libs/sgeobj/sge_cqueue.c
sge-8.1.8/source/libs/sgeobj/sge_cqueue.c
--- sge-8.1.8-orig/source/libs/sgeobj/sge_cqueue.c 2014-10-23
18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_cqueue.c 2015-03-31 14:18:00.195685975
-0400
@@ -113,7 +113,8 @@
{ CQ_h_rss, QU_h_rss, AMEM_href,
AMEM_value, NoName, SGE_ATTR_H_RSS, false, false,
cqueue_verify_memory_value},
{ CQ_s_vmem, QU_s_vmem, AMEM_href,
AMEM_value, NoName, SGE_ATTR_S_VMEM, false, false,
cqueue_verify_memory_value},
{ CQ_h_vmem, QU_h_vmem, AMEM_href,
AMEM_value, NoName, SGE_ATTR_H_VMEM, false, false,
cqueue_verify_memory_value},
-
+ { CQ_s_as, QU_s_as, AMEM_href,
AMEM_value, NoName, SGE_ATTR_S_AS, false, false,
cqueue_verify_memory_value},
+ { CQ_h_as, QU_h_as, AMEM_href,
AMEM_value, NoName, SGE_ATTR_H_AS, false, false,
cqueue_verify_memory_value},
{ CQ_s_rt, QU_s_rt, ATIME_href,
ATIME_value, NoName, SGE_ATTR_S_RT, false, false,
cqueue_verify_time_value},
{ CQ_h_rt, QU_h_rt, ATIME_href,
ATIME_value, NoName, SGE_ATTR_H_RT, false, false,
cqueue_verify_time_value},
{ CQ_s_cpu, QU_s_cpu, ATIME_href,
ATIME_value, NoName, SGE_ATTR_S_CPU, false, false,
cqueue_verify_time_value},
@@ -639,12 +640,14 @@
"INFINITY", "INFINITY", "INFINITY", "INFINITY",
"INFINITY", "INFINITY", "INFINITY", "INFINITY",
"INFINITY", "INFINITY", "INFINITY", "INFINITY",
+ "INFINITY", "INFINITY",
NULL
};
const int attr[] = {
CQ_s_fsize, CQ_h_fsize, CQ_s_data, CQ_h_data,
CQ_s_stack, CQ_h_stack, CQ_s_core, CQ_h_core,
CQ_s_rss, CQ_h_rss, CQ_s_vmem, CQ_h_vmem,
+ CQ_s_as, CQ_h_as,
NoName
};
int index = 0;
diff -ruN sge-8.1.8-orig/source/libs/sgeobj/sge_cqueue.h
sge-8.1.8/source/libs/sgeobj/sge_cqueue.h
--- sge-8.1.8-orig/source/libs/sgeobj/sge_cqueue.h 2014-10-23
18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_cqueue.h 2015-03-31 14:18:00.207133157
-0400
@@ -164,6 +164,8 @@
#define SGE_ATTR_S_RSS "s_rss"
#define SGE_ATTR_H_VMEM "h_vmem"
#define SGE_ATTR_S_VMEM "s_vmem"
+#define SGE_ATTR_H_AS "h_as"
+#define SGE_ATTR_S_AS "s_as"
#define SGE_ATTR_QTYPE "qtype"
#define SGE_ATTR_SEQ_NO "seq_no"
diff -ruN sge-8.1.8-orig/source/libs/sgeobj/sge_cqueue_CQ_L.h
sge-8.1.8/source/libs/sgeobj/sge_cqueue_CQ_L.h
--- sge-8.1.8-orig/source/libs/sgeobj/sge_cqueue_CQ_L.h 2014-10-23
18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_cqueue_CQ_L.h 2015-03-31
14:18:00.201336491 -0400
@@ -63,6 +63,8 @@
CQ_h_rss,
CQ_s_vmem,
CQ_h_vmem,
+ CQ_s_as,
+ CQ_h_as,
CQ_s_rt,
CQ_h_rt,
@@ -132,6 +134,8 @@
SGE_MAP_D(CQ_h_rss, AMEM_Type, CULL_SPOOL | CULL_JGDI_CONF, "@/",
"INFINITY", "hostname", "hRss")
SGE_MAP_D(CQ_s_vmem, AMEM_Type, CULL_SPOOL | CULL_JGDI_CONF, "@/",
"INFINITY", "hostname", "sVmem")
SGE_MAP_D(CQ_h_vmem, AMEM_Type, CULL_SPOOL | CULL_JGDI_CONF, "@/",
"INFINITY", "hostname", "hVmem")
+ SGE_MAP_D(CQ_s_as, AMEM_Type, CULL_SPOOL | CULL_JGDI_CONF, "@/",
"INFINITY", "hostname", "sAs")
+ SGE_MAP_D(CQ_h_as, AMEM_Type, CULL_SPOOL | CULL_JGDI_CONF, "@/",
"INFINITY", "hostname", "hAs")
SGE_MAP_D(CQ_s_rt, ATIME_Type, CULL_SPOOL | CULL_JGDI_CONF, "@/",
"INFINITY", "hostname", "sRt")
SGE_MAP_D(CQ_h_rt, ATIME_Type, CULL_SPOOL | CULL_JGDI_CONF, "@/",
"INFINITY", "hostname", "hRt")
@@ -199,6 +203,8 @@
NAME("CQ_h_rss")
NAME("CQ_s_vmem")
NAME("CQ_h_vmem")
+ NAME("CQ_s_as")
+ NAME("CQ_h_as")
NAME("CQ_s_rt")
NAME("CQ_h_rt")
diff -ruN sge-8.1.8-orig/source/libs/sgeobj/sge_qinstance_QU_L.h
sge-8.1.8/source/libs/sgeobj/sge_qinstance_QU_L.h
--- sge-8.1.8-orig/source/libs/sgeobj/sge_qinstance_QU_L.h 2014-10-23
18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_qinstance_QU_L.h 2015-03-31
14:18:00.213553637 -0400
@@ -86,6 +86,8 @@
QU_h_rss,
QU_s_vmem,
QU_h_vmem,
+ QU_s_as,
+ QU_h_as,
QU_suspend_interval,
QU_min_cpu_interval,
@@ -178,6 +180,8 @@
SGE_STRING(QU_h_rss, CULL_CONFIGURE | CULL_JGDI_RO)
SGE_STRING(QU_s_vmem, CULL_CONFIGURE | CULL_JGDI_RO)
SGE_STRING(QU_h_vmem, CULL_CONFIGURE | CULL_JGDI_RO)
+ SGE_STRING(QU_s_as, CULL_CONFIGURE | CULL_JGDI_RO)
+ SGE_STRING(QU_h_as, CULL_CONFIGURE | CULL_JGDI_RO)
SGE_STRING(QU_suspend_interval, CULL_CONFIGURE | CULL_JGDI_RO)
SGE_STRING(QU_min_cpu_interval, CULL_CONFIGURE | CULL_JGDI_RO)
@@ -265,6 +269,8 @@
NAME("QU_h_rss")
NAME("QU_s_vmem")
NAME("QU_h_vmem")
+ NAME("QU_s_as")
+ NAME("QU_h_as")
NAME("QU_suspend_interval")
NAME("QU_min_cpu_interval")
diff -ruN sge-8.1.8-orig/source/libs/spool/flatfile/sge_flatfile_obj.c
sge-8.1.8/source/libs/spool/flatfile/sge_flatfile_obj.c
--- sge-8.1.8-orig/source/libs/spool/flatfile/sge_flatfile_obj.c
2014-10-23 18:03:58.000000000 -0400
+++ sge-8.1.8/source/libs/spool/flatfile/sge_flatfile_obj.c 2015-03-31
14:18:00.219086079 -0400
@@ -482,6 +482,8 @@
{ CQ_s_rss, 21, "s_rss", AMEM_sub_fields,
&qconf_sub_name_value_comma_braced_sfi, read_CQ_mem_attr_list,
write_CQ_mem_attr_list},
{ CQ_h_rss, 21, "h_rss", AMEM_sub_fields,
&qconf_sub_name_value_comma_braced_sfi, read_CQ_mem_attr_list,
write_CQ_mem_attr_list},
{ CQ_s_vmem, 21, "s_vmem", AMEM_sub_fields,
&qconf_sub_name_value_comma_braced_sfi, read_CQ_mem_attr_list,
write_CQ_mem_attr_list},
+ { CQ_s_as, 21, "s_as", AMEM_sub_fields,
&qconf_sub_name_value_comma_braced_sfi, read_CQ_mem_attr_list,
write_CQ_mem_attr_list},
+ { CQ_h_as, 21, "h_as", AMEM_sub_fields,
&qconf_sub_name_value_comma_braced_sfi, read_CQ_mem_attr_list,
write_CQ_mem_attr_list},
{ CQ_h_vmem, 21, "h_vmem", AMEM_sub_fields,
&qconf_sub_name_value_comma_braced_sfi, read_CQ_mem_attr_list,
write_CQ_mem_attr_list},
{ NoName, 21, NULL, NULL,
NULL, NULL, NULL}
};
@@ -967,8 +969,9 @@
spooling_field *sge_build_QU_field_list(bool to_stdout, bool to_file)
{
- /* There are 52 possible QU_Type fields. */
- spooling_field *fields = (spooling_field
*)malloc(sizeof(spooling_field)*52);
+ /* There are 54 possible QU_Type fields. */
+ spooling_field *fields = (spooling_field
*)malloc(sizeof(spooling_field)*54);
+
int count = 0;
create_spooling_field (&fields[count++], QU_qname, 21, "qname", NULL,
@@ -1082,6 +1085,10 @@
NULL, NULL, NULL);
create_spooling_field (&fields[count++], QU_h_vmem, 21, "h_vmem", NULL,
NULL, NULL, NULL);
+ create_spooling_field (&fields[count++], QU_s_as, 21, "s_as", NULL,
+ NULL, NULL, NULL);
+ create_spooling_field (&fields[count++], QU_h_as, 21, "h_as", NULL,
+ NULL, NULL, NULL);
} else if (to_file) {
/*
* Spool only non-CQ attributes
diff -ruN sge-8.1.8-orig/source/libs/uti/sge_stdio.h
sge-8.1.8/source/libs/uti/sge_stdio.h
--- sge-8.1.8-orig/source/libs/uti/sge_stdio.h 2014-10-23 18:03:59.000000000
-0400
+++ sge-8.1.8/source/libs/uti/sge_stdio.h 2015-03-31 14:18:00.228596000
-0400
@@ -49,6 +49,33 @@
goto FOPEN_ERROR; \
}
+/****** uti/stdio/FSCANF() ***************************************************
+ * * NAME
+ * * FSCANF() -- fscanf() macro
+ * *
+ * * SYNOPSIS
+ * * #define FSCANF(arguments)
+ * * void fscanf(FILE *stream, const char *format, ...)
+ * *
+ * * FUNCTION
+ * * This FSCANF macro has to be used similar to the fscanf
+ * * function. It is not necessary to check the return value.
+ * * In case of an error the macro will jump to a defined label.
+ * * The label name is 'FSCANF_ERROR'.
+ * *
+ * * INPUTS
+ * * FILE *stream - output stream
+ * * const char *format - format string
+ * * ...
+ * *
+ * * NOTES
+ * * Don't forget to define the 'FSCANF_ERROR'-label
+ *
******************************************************************************/
+#define FSCANF(x) \
+ if (fscanf x < 0) { \
+ goto FSCANF_ERROR; \
+ }
+
/****** uti/stdio/FPRINTF() ***************************************************
* NAME
* FPRINTF() -- fprintf() macro
diff -ruN sge-8.1.8-cgroups/doc/man/man5/sge_conf.5
sge-8.1.8/doc/man/man5/sge_conf.5
--- sge-8.1.8-cgroups/doc/man/man5/sge_conf.5 2015-03-31 14:18:00.010993411
-0400
+++ sge-8.1.8/doc/man/man5/sge_conf.5 2015-03-31 14:22:51.305493493 -0400
@@ -1559,6 +1559,10 @@
.I sources/experimental/qidle
in the source repository, but it may be better to check the
the screensaver state.
+.IP \fIENABLE_USEDMEMORY\fP
+If this parameter is set to true, then xxQS_NAMExx will use measure
+virtual memory usage (vmem) as the sum of physical memory and swap
+occupied by jobs.
.IP \fICGROUP_MEMORY\fP
If this parameter is set to a directory, then xxQS_NAMExx will create a
cgroup in that location for each job instance running on the execd. It
diff -ruN sge-8.1.8-cgroups/source/daemons/common/procfs.c
sge-8.1.8/source/daemons/common/procfs.c
--- sge-8.1.8-cgroups/source/daemons/common/procfs.c 2014-10-23
18:03:56.000000000 -0400
+++ sge-8.1.8/source/daemons/common/procfs.c 2015-03-31 14:22:51.308308744
-0400
@@ -80,6 +80,7 @@
#include "cull/cull.h"
#include "basis_types.h"
+#include "sge_conf.h"
#include "sgedefs.h"
#include "exec_ifm.h"
#include "pdc.h"
@@ -103,6 +104,8 @@
unsigned long *stime, unsigned long *vsize);
#endif
+static bool enable_usedmemory = false;
+
/*-----------------------------------------------------------------------*/
static gid_t *list = 0;
#if defined(LINUX) || defined(ALPHA) || defined(SOLARIS)
@@ -336,6 +339,8 @@
int pt_open(void)
{
+ enable_usedmemory = mconf_get_enable_usedmemory();
+
cwd = opendir(PROC_DIR);
return !cwd;
}
@@ -399,6 +404,54 @@
if (!(linux_read_status(pidname, time_stamp, job_list,
&pid, &utime, &stime, &vsize) != 0))
goto return0;
+
+ /*
+ * Obtain accurate memory usage statistics, if requested
+ * (instead of virtual memory).
+ */
+ if (enable_usedmemory) {
+ vsize = 0;
+ unsigned long value = 0;
+ int found_swap = 0;
+
+ FILE *fp;
+ char membuf[BIGLINE];
+ char key[10]; /* Size must match width in sscanf */
+
+ snprintf(membuf, sizeof(membuf), PROC_DIR "/%s/status",
dent->d_name);
+ if ((fp = fopen(membuf, "r"))) {
+ while(fgets(membuf, sizeof(membuf), fp)) {
+ if (sscanf(membuf, "%10s %lu", key, &value) == 2) {
+ if (strncmp(membuf, "VmRSS:", 6) == 0) {
+ vsize += value * 1024;
+ } else if(strncmp(membuf, "VmSwap:", 7) == 0) {
+ vsize += value * 1024;
+ found_swap++;
+ }
+ }
+ }
+ fclose(fp);
+ }
+
+ /* Older Linux kernels keep swap usage elsewhere, */
+ /* where values from each map need to be added up. */
+ if (found_swap == 0) {
+ snprintf(membuf, sizeof(membuf), PROC_DIR "/%s/smaps",
dent->d_name);
+ if ((fp = fopen(membuf, "r"))) {
+ while(fgets(membuf, sizeof(membuf), fp)) {
+ if (sscanf(membuf, "%10s %lu", key, &value) == 2) {
+ if (strncmp(membuf, "Swap:", 5) == 0) {
+ vsize += value * 1024;
+ found_swap++;
+ }
+ }
+ }
+ fclose(fp);
+ }
+ }
+ }
+
+
if (pr == NULL) {
pr = lCreateElem(PRO_Type);
lSetPosUlong(pr, pos_pid, pid);
diff -ruN sge-8.1.8-cgroups/source/libs/sgeobj/sge_conf.c
sge-8.1.8/source/libs/sgeobj/sge_conf.c
--- sge-8.1.8-cgroups/source/libs/sgeobj/sge_conf.c 2015-03-31
14:18:00.176952557 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_conf.c 2015-03-31 14:22:51.322200878
-0400
@@ -190,6 +190,7 @@
static bool keep_active = false;
static bool enable_windomacc = false;
static bool enable_binding = false;
+static bool enable_usedmemory = false;
static char* cgroup_memory = NULL;
static bool enable_addgrp_kill = true;
static u_long32 pdc_interval = 1;
@@ -875,6 +876,7 @@
keep_active = false;
enable_windomacc = false;
enable_binding = HAVE_HWLOC ? true : false;
+ enable_usedmemory = false;
sge_free(&cgroup_memory);
enable_addgrp_kill = true;
use_qsub_gid = false;
@@ -935,6 +937,9 @@
if (parse_bool_param(s, "ENABLE_BINDING", &enable_binding)) {
continue;
}
+ if (parse_bool_param(s, "ENABLE_USEDMEMORY", &enable_usedmemory)) {
+ continue;
+ }
if (!strncasecmp(s, "CGROUP_MEMORY", sizeof("CGROUP_MEMORY")-1)) {
cgroup_memory = sge_strdup(NULL, &(s[sizeof("CGROUP_MEMORY")]));
continue;
@@ -2104,6 +2109,18 @@
DRETURN(ret);
}
+bool mconf_get_enable_usedmemory(void) {
+ bool ret;
+
+ DENTER(BASIS_LAYER, "mconf_get_enable_usedmemory");
+ SGE_LOCK(LOCK_MASTER_CONF, LOCK_READ);
+
+ ret = enable_usedmemory;
+
+ SGE_UNLOCK(LOCK_MASTER_CONF, LOCK_READ);
+ DRETURN(ret);
+}
+
/* returned pointer needs to be freed */
char* mconf_get_cgroup_memory(void) {
char* ret = NULL;
diff -ruN sge-8.1.8-cgroups/source/libs/sgeobj/sge_conf.h
sge-8.1.8/source/libs/sgeobj/sge_conf.h
--- sge-8.1.8-cgroups/source/libs/sgeobj/sge_conf.h 2015-03-31
14:18:00.186024000 -0400
+++ sge-8.1.8/source/libs/sgeobj/sge_conf.h 2015-03-31 14:22:51.332964000
-0400
@@ -117,6 +117,7 @@
bool mconf_get_keep_active(void);
bool mconf_get_enable_windomacc(void);
bool mconf_get_enable_binding(void);
+bool mconf_get_enable_usedmemory(void);
char* mconf_get_cgroup_memory(void);
bool mconf_get_simulate_execds(void);
bool mconf_get_simulate_jobs(void);
_______________________________________________
SGE-discuss mailing list
[email protected]
https://arc.liv.ac.uk/mailman/listinfo/sge-discuss
