Dear Support,
there is a problem with OpenMPI in version 1.3 and version 1.3.1 when
using our batch system Slurm. On our parallel computer there are 2
queues - one with exclusive usage of slots (cores) (SLURM_OVERCOMMIT=0)
within nodes and one without shared usage of slots (SLURM_OVERCOMMIT=1)
within nodes. Running a simple MPI-program (I'll send you this program
mpi_hello.c as attachment) with SLURM_OVERCOMMIT set to 0 the
executable works fine, running it with SLURM_OVERCOMMIT set to 1 it does
not work correctly. Please have a look to 2 files with outputs.
Working not correctly means that the MPI-program runs on 1 processor
although I have started it (for example) on 4 processors (it does not
work correctly for any processor number not equal to 1).
This error only occurs for the version 1.3 and 1.3.1. If I am using oder
versions of OpenMPI the program works fine.
In the file Job_101442.out the hostlist (4x icn001) from Slurm is
printed, then the content of the file
/scratch/JMS_tmpdir/Job_101442/tmp.CCaCM22772 is printed, then the
commandline (mpirun ...) is printed, then stdout of the program
mpi_hello.c is printed (it runs only on 1 processor!!!) and the
environment is printed.
In the file Job_101440.out the same program is run. The only difference
is, that SLURM_OVERCOMMIT is'nt set!
Under the hood of job_submit .... salloc -n4 script is started.
In "script" you find the command
mpirun --hostfile ..... as you can see in both output files.
Sincerly yours
H. Häfner
--
Hartmut Häfner
Karlsruhe Institute of Technology (KIT)
University Karlsruhe (TH)
Steinbuch Centre for Computing (SCC)
Scientific Computing and Applications (SCA)
Zirkel 2 (Campus Süd, Geb. 20.21, Raum 204)
D-76128 Karlsruhe
Fon +49(0)721 608 4869
Fax +49(0)721 32550
hartmut.haef...@kit.edu
http://www.rz.uni-karlsruhe.de/personen/hartmut.haefner
#include <cstdio>
#include <mpi.h>
int main(int argc, char **argv)
{
MPI_Init(&argc, &argv);
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
int size;
MPI_Comm_size(MPI_COMM_WORLD, &size);
printf("size = %d\n", size);
printf("rank = %d\n", rank);
system("env | grep SLURM");
system("env | grep JMS");
MPI_Finalize();
}
Universitaet Karlsruhe / Rechenzentrum +++ ic1 +++ JMS 2.1 / slurm 1.2.22
Job 101442 has been submitted at 19.03.2009/17:38:27 on ic1n990
Command: job_submit -t 10 -m 2000 -c d -p 4 mpirun mpi_helloc
Job 101442 started at 19.03.2009/17:38:29 on ic1n001
4 tasks running on 1 nodes overcommited: 4*ic1n001
================================================================================
ic1n001
ic1n001
ic1n001
ic1n001
ic1n001 max_slots=4
/software/all/openmpi/1.3.1/bin/mpirun --hostfile
/scratch/JMS_tmpdir/Job_101442/tmp.CCaCM22772 /jms/home/jobscript/run mpi_helloc
size = 1
rank = 0
SLURM_NODELIST=ic1n001
SLURMD_NODENAME=ic1n001
SLURM_PRIO_PROCESS=0
SLURM_WAIT=30
SLURM_NNODES=1
SLURM_JOBID=101442
SLURM_TASKS_PER_NODE=1
SLURM_PARTITION=d
SLURM_CPUS_PER_TASK=1
SLURM_JOB_ID=101442
SLURM_UMASK=0022
SLURM_NODEID=0
SLURM_OVERCOMMIT=1
SLURM_NPROCS=4
SLURM_TASK_PID=22700
SLURM_PROCID=0
SLURM_JOB_NODELIST=ic1n001
SLURM_LOCALID=0
SLURM_JOB_CPUS_PER_NODE=1
SLURM_GTIDS=0
SLURM_TIMELIMIT=20
SLURM_JOB_NUM_NODES=1
JMS_test_quota=n
JMS_c=d
JMS_nodes=ic1n001 ic1n001 ic1n001 ic1n001
JMS_slurm_wait=30
JMS_jid=101442
JMS_node0=ic1n001
JMS_org=kit
JMS_root=/jms/home
JMS_e=Job_%j.err
JMS_submit_time=1237480707
JMS_sysname=ic1
JMS_start_time=1237480708
JMS_i=/dev/null
JMS_exclusive=2
JMS_last_start_time=
JMS_o=Job_%j.out
JMS_last_jid=
JMS_event_end=0
JMS_m=2000
TEMP=/scratch/JMS_tmpdir/Job_101442
JMS_p=4
JMS_nodes_list=ic1n001
JMS_t=10
JMS_prio=1
JMS_threads=1
JMS_systime=1237480708
JMS_group=rz00
JMS_pwd=/home/ws/rz50/softtest
JMS_submit_command=-t 10 -m 2000 -c d -p 4 mpirun mpi_helloc
JMS_user=rz50
JMS_systype=3
JMS_submit_node=ic1n990
JMS_A=-
JMS_Nnodes=1
JMS_event_begin=0
JMS_J=
JMS_node_used=4*ic1n001
JMS_stdin=/dev/null
JMS_job=mpirun mpi_helloc
JMS_version=2.1
JMS_tasks=4
TMP=/scratch/JMS_tmpdir/Job_101442
MPI_TMPDIR=/scratch/JMS_tmpdir/Job_101442
JMS_stdout=Job_101442.out
JMS_T=20
JMS_fat_nodes=n
JMS_stderr=Job_101442.err
JMS_test_project=n
JMS_trace_level=0
JMS_T_diff=0
================================================================================
Job 101442 completed at 19.03.2009/17:38:30 (COMPLETED)
Universitaet Karlsruhe / Rechenzentrum +++ ic1 +++ JMS 2.1 / slurm 1.2.22
Job 101440 has been submitted at 19.03.2009/17:30:55 on ic1n990
Command: job_submit -t 10 -m 2000 -c p -p 4 mpirun mpi_helloc
Job 101440 started at 19.03.2009/17:30:57 on ic1n029
4 tasks running on 1 nodes shared: 4*ic1n029
================================================================================
ic1n029
ic1n029
ic1n029
ic1n029
ic1n029 max_slots=4
/software/all/openmpi/1.3.1/bin/mpirun --hostfile
/scratch/JMS_tmpdir/Job_101440/tmp.KdppsV3849 /jms/home/jobscript/run mpi_helloc
size = 4
rank = 0
size = 4
rank = 3
size = 4
rank = 2
size = 4
rank = 1
SLURM_NODELIST=ic1n029
SLURM_NODELIST=ic1n029
SLURM_NODELIST=ic1n029
SLURMD_NODENAME=ic1n029
SLURM_PRIO_PROCESS=0
SLURMD_NODENAME=ic1n029
SLURM_PRIO_PROCESS=0
SLURMD_NODENAME=ic1n029
SLURM_PRIO_PROCESS=0
SLURM_WAIT=30
SLURM_NNODES=1
SLURM_JOBID=101440
SLURM_TASKS_PER_NODE=4
SLURM_PARTITION=t
SLURM_CPUS_PER_TASK=1
SLURM_JOB_ID=101440
SLURM_WAIT=30
SLURM_NNODES=1
SLURM_JOBID=101440
SLURM_WAIT=30
SLURM_NNODES=1
SLURM_JOBID=101440
SLURM_TASKS_PER_NODE=4
SLURM_PARTITION=t
SLURM_CPUS_PER_TASK=1
SLURM_JOB_ID=101440
SLURM_UMASK=0022
SLURM_NODEID=0
SLURM_NPROCS=4
SLURM_TASK_PID=3777
SLURM_PROCID=0
SLURM_TASKS_PER_NODE=4
SLURM_PARTITION=t
SLURM_CPUS_PER_TASK=1
SLURM_JOB_ID=101440
SLURM_UMASK=0022
SLURM_NODEID=0
SLURM_NPROCS=4
SLURM_JOB_NODELIST=ic1n029
SLURM_LOCALID=0
SLURM_JOB_CPUS_PER_NODE=4
SLURM_GTIDS=0
SLURM_UMASK=0022
SLURM_NODEID=0
SLURM_NPROCS=4
SLURM_TASK_PID=3777
SLURM_PROCID=0
SLURM_TASK_PID=3777
SLURM_PROCID=0
SLURM_JOB_NODELIST=ic1n029
SLURM_LOCALID=0
SLURM_TIMELIMIT=11
SLURM_JOB_CPUS_PER_NODE=4
SLURM_GTIDS=0
SLURM_TIMELIMIT=11
SLURM_JOB_NODELIST=ic1n029
SLURM_LOCALID=0
SLURM_JOB_CPUS_PER_NODE=4
SLURM_GTIDS=0
SLURM_TIMELIMIT=11
SLURM_JOB_NUM_NODES=1
SLURM_JOB_NUM_NODES=1
SLURM_JOB_NUM_NODES=1
SLURM_NODELIST=ic1n029
SLURMD_NODENAME=ic1n029
SLURM_PRIO_PROCESS=0
SLURM_WAIT=30
SLURM_NNODES=1
SLURM_JOBID=101440
SLURM_TASKS_PER_NODE=4
SLURM_PARTITION=t
SLURM_CPUS_PER_TASK=1
SLURM_JOB_ID=101440
SLURM_UMASK=0022
SLURM_NODEID=0
SLURM_NPROCS=4
SLURM_TASK_PID=3777
SLURM_PROCID=0
SLURM_JOB_NODELIST=ic1n029
SLURM_LOCALID=0
SLURM_JOB_CPUS_PER_NODE=4
SLURM_GTIDS=0
SLURM_TIMELIMIT=11
SLURM_JOB_NUM_NODES=1
JMS_test_quota=n
JMS_c=p
JMS_nodes=ic1n029 ic1n029 ic1n029 ic1n029
JMS_slurm_wait=30
JMS_jid=101440
JMS_node0=ic1n029
JMS_org=kit
JMS_root=/jms/home
JMS_e=Job_%j.err
JMS_submit_time=1237480255
JMS_sysname=ic1
JMS_start_time=1237480255
JMS_i=/dev/null
JMS_exclusive=0
JMS_last_start_time=
JMS_o=Job_%j.out
JMS_last_jid=
JMS_event_end=0
JMS_m=2000
TEMP=/scratch/JMS_tmpdir/Job_101440
JMS_p=4
JMS_nodes_list=ic1n029
JMS_t=10
JMS_prio=1
JMS_threads=1
JMS_systime=1237480255
JMS_group=rz00
JMS_pwd=/home/ws/rz50/softtest
JMS_submit_command=-t 10 -m 2000 -c p -p 4 mpirun mpi_helloc
JMS_user=rz50
JMS_systype=3
JMS_submit_node=ic1n990
JMS_A=-
JMS_Nnodes=1
JMS_event_begin=0
JMS_J=
JMS_node_used=4*ic1n029
JMS_stdin=/dev/null
JMS_job=mpirun mpi_helloc
JMS_version=2.1
JMS_tasks=4
TMP=/scratch/JMS_tmpdir/Job_101440
MPI_TMPDIR=/scratch/JMS_tmpdir/Job_101440
JMS_stdout=Job_101440.out
JMS_T=11
JMS_fat_nodes=n
JMS_stderr=Job_101440.err
JMS_test_project=n
JMS_trace_level=0
JMS_T_diff=0
JMS_test_quota=n
JMS_c=p
JMS_nodes=ic1n029 ic1n029 ic1n029 ic1n029
JMS_slurm_wait=30
JMS_jid=101440
JMS_node0=ic1n029
JMS_org=kit
JMS_root=/jms/home
JMS_e=Job_%j.err
JMS_submit_time=1237480255
JMS_sysname=ic1
JMS_start_time=1237480255
JMS_i=/dev/null
JMS_exclusive=0
JMS_last_start_time=
JMS_o=Job_%j.out
JMS_last_jid=
JMS_event_end=0
JMS_m=2000
TEMP=/scratch/JMS_tmpdir/Job_101440
JMS_p=4
JMS_nodes_list=ic1n029
JMS_t=10
JMS_prio=1
JMS_threads=1
JMS_systime=1237480255
JMS_group=rz00
JMS_pwd=/home/ws/rz50/softtest
JMS_submit_command=-t 10 -m 2000 -c p -p 4 mpirun mpi_helloc
JMS_user=rz50
JMS_systype=3
JMS_submit_node=ic1n990
JMS_A=-
JMS_Nnodes=1
JMS_event_begin=0
JMS_J=
JMS_node_used=4*ic1n029
JMS_stdin=/dev/null
JMS_job=mpirun mpi_helloc
JMS_version=2.1
JMS_tasks=4
TMP=/scratch/JMS_tmpdir/Job_101440
MPI_TMPDIR=/scratch/JMS_tmpdir/Job_101440
JMS_stdout=Job_101440.out
JMS_T=11
JMS_fat_nodes=n
JMS_stderr=Job_101440.err
JMS_test_project=n
JMS_trace_level=0
JMS_T_diff=0
JMS_test_quota=n
JMS_c=p
JMS_nodes=ic1n029 ic1n029 ic1n029 ic1n029
JMS_slurm_wait=30
JMS_jid=101440
JMS_node0=ic1n029
JMS_org=kit
JMS_root=/jms/home
JMS_e=Job_%j.err
JMS_submit_time=1237480255
JMS_sysname=ic1
JMS_start_time=1237480255
JMS_i=/dev/null
JMS_exclusive=0
JMS_last_start_time=
JMS_o=Job_%j.out
JMS_last_jid=
JMS_event_end=0
JMS_m=2000
TEMP=/scratch/JMS_tmpdir/Job_101440
JMS_p=4
JMS_nodes_list=ic1n029
JMS_t=10
JMS_prio=1
JMS_threads=1
JMS_systime=1237480255
JMS_group=rz00
JMS_pwd=/home/ws/rz50/softtest
JMS_submit_command=-t 10 -m 2000 -c p -p 4 mpirun mpi_helloc
JMS_user=rz50
JMS_systype=3
JMS_submit_node=ic1n990
JMS_A=-
JMS_Nnodes=1
JMS_event_begin=0
JMS_J=
JMS_node_used=4*ic1n029
JMS_stdin=/dev/null
JMS_job=mpirun mpi_helloc
JMS_version=2.1
JMS_tasks=4
TMP=/scratch/JMS_tmpdir/Job_101440
MPI_TMPDIR=/scratch/JMS_tmpdir/Job_101440
JMS_stdout=Job_101440.out
JMS_T=11
JMS_fat_nodes=n
JMS_stderr=Job_101440.err
JMS_test_project=n
JMS_trace_level=0
JMS_T_diff=0
JMS_test_quota=n
JMS_c=p
JMS_nodes=ic1n029 ic1n029 ic1n029 ic1n029
JMS_slurm_wait=30
JMS_jid=101440
JMS_node0=ic1n029
JMS_org=kit
JMS_root=/jms/home
JMS_e=Job_%j.err
JMS_submit_time=1237480255
JMS_sysname=ic1
JMS_start_time=1237480255
JMS_i=/dev/null
JMS_exclusive=0
JMS_last_start_time=
JMS_o=Job_%j.out
JMS_last_jid=
JMS_event_end=0
JMS_m=2000
TEMP=/scratch/JMS_tmpdir/Job_101440
JMS_p=4
JMS_nodes_list=ic1n029
JMS_t=10
JMS_prio=1
JMS_threads=1
JMS_systime=1237480255
JMS_group=rz00
JMS_pwd=/home/ws/rz50/softtest
JMS_submit_command=-t 10 -m 2000 -c p -p 4 mpirun mpi_helloc
JMS_user=rz50
JMS_systype=3
JMS_submit_node=ic1n990
JMS_A=-
JMS_Nnodes=1
JMS_event_begin=0
JMS_J=
JMS_node_used=4*ic1n029
JMS_stdin=/dev/null
JMS_job=mpirun mpi_helloc
JMS_version=2.1
JMS_tasks=4
TMP=/scratch/JMS_tmpdir/Job_101440
MPI_TMPDIR=/scratch/JMS_tmpdir/Job_101440
JMS_stdout=Job_101440.out
JMS_T=11
JMS_fat_nodes=n
JMS_stderr=Job_101440.err
JMS_test_project=n
JMS_trace_level=0
JMS_T_diff=0
================================================================================
Job 101440 completed at 19.03.2009/17:31:00 (COMPLETED)
