Hi,
in the meantime I found some useful pieces of code within the sge sources:
~/sge-8.1.9/source/dist/util/resources/loadsensors
gpu-loadsensor.c
The Makefile did not work for me. Maybe the attached patch is useful for
others too.
I changed also a little bit the c-file to get the utilization/free gpus.
I did this under the assumption that the processes run in
--compute-mode=EXCLUSIVE_PROCESS on the gpu.
qhost -F gpu.cuda.procsum gpu.cuda.procfreexcl
HOSTNAME ARCH NCPU NSOC NCOR NTHR LOAD MEMTOT
MEMUSE SWAPTO SWAPUS
----------------------------------------------------------------------------------------------
global - - - - - - -
- - -
alpha lx-amd64 40 2 20 40 0.01 251.8G
4.8G 869.8G 614.3M
gamma lx-amd64 24 2 12 24 3.67 125.8G
6.5G 238.0G 51.6M
Host Resource(s): hl:gpu.cuda.procsum=4.000000
hl:gpu.cuda.procfreexcl=0.000000
omega lx-amd64 24 2 12 24 5.14 251.9G
11.0G 20.0G 1.1G
Host Resource(s): hl:gpu.cuda.procsum=0.000000
hc:gpu.cuda.procfreexcl=1.000000
Regards,
Thomas
--- gpu-loadsensor.c.original 2016-07-22 14:58:48.522223609 +0200
+++ gpu-loadsensor.c 2016-07-22 16:30:54.696002404 +0200
@@ -406,7 +406,10 @@
print_nvml (void)
{
unsigned int i;
-
+ unsigned int uintsum;
+ unsigned int uintdif;
+ uintsum = 0;
+ uintdif = 0;
for (i = 0; i < n_cuda; i++) {
nvmlDevice_t dev;
char name[80];
@@ -435,6 +438,7 @@
infos))
== 0)
printf ("%s:gpu.cuda.%u.procs:%u\n", host, i, uint);
+ uintsum = uintsum + uint;
#endif
#if NVML_API_VERSION >= 2
/* Otherwise, we only have the current clock speed, which isn't
@@ -449,6 +453,13 @@
== 0)
printf ("%s:gpu.cuda.%u.util:%u\n", host, i, utilization.gpu);
}
+
+ printf ("%s:gpu.cuda.procsum:%u\n", host, uintsum);
+ if ( uintsum <= n_cuda) {
+ uintdif = n_cuda - uintsum;
+ }
+ printf ("%s:gpu.cuda.procfreexcl:%u\n", host, uintdif);
+
}
#endif
--
There’s no place like ::1
Thomas Schäfer (Systemverwaltung)
Ludwig-Maximilians-Universität
Centrum für Informations- und Sprachverarbeitung
Oettingenstraße 67 Raum C109
80538 München ☎ +49/89/2180-9706 ℻ +49/89/2180-9701
--- Makefile.org 2016-07-19 22:30:01.933488084 +0200
+++ Makefile 2016-07-19 22:48:59.021213449 +0200
@@ -1,16 +1,16 @@
CFLAGS = -g
# Remove either of these, as appropriate, and add -I for nvml.h and CL/cl.h
-CPPFLAGS = -DHAVE_NVML=1 -DHAVE_OPENCL=1
+CPPFLAGS = -DHAVE_NVML=1 -DHAVE_OPENCL=1 -I/usr/local/cuda/include
# Add -L for -lOpenCL and -lnvidia-ml. Put ATI path (if any) first to
# pick up that lOpenCL rather than the CUDA one.
-LDFLAGS =
+LDFLAGS = -L/usr/lib64 -L/usr/local/cuda/lib64
# Remove either, as appropriate.
LIBS = -lOpenCL -lnvidia-ml
.PHONY: clean
gpu-loadsensor: gpu-loadsensor.c
- $(CC) $(CPPFLAGS) $(CFLAGS) -o gpu-loadsensor gpu-loadsensor.c $(LDFLAGS)
+ $(CC) $(CPPFLAGS) $(CFLAGS) -o gpu-loadsensor gpu-loadsensor.c $(LIBS) $(LDFLAGS)
clean:
rm -f gpu-loadsensor *.o
_______________________________________________
SGE-discuss mailing list
[email protected]
https://arc.liv.ac.uk/mailman/listinfo/sge-discuss
