Thanks for responding. I used a linux cluster. I think I would like to create a
model that is multithreaded and each thread can make MPI calls. I attached test
code as follow. It has two pthreads and there are MPI calls in both of those
two threads. In the main function, there are also MPI calls. Should I use a
full multithreading? Thanks again.
#include <iostream>
#include <pthread.h>
#include <fstream>
#include <sstream>
#include <string.h>
#include "mpi.h"
using namespace std;
pthread_mutex_t _dealmutex;
pthread_mutex_t _dealmutex1;
pthread_mutex_t _dealmutex2;
void* backID(void* arg)
{
int myid;
pthread_mutex_init(&_dealmutex1, NULL);
stringstream RANK;
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
RANK << myid;
cout << myid << " create background ID" << endl;
int v;
MPI_Status status;
MPI_Request requ1, requ2;
int m;
int x, y;
int count = 0;
string filename("f_");
filename += RANK.str();
filename += "_backID.txt";
fstream fout(filename.c_str(), ios::out);
if(!fout)
{
cout << "can not create the file " << filename << endl;
fout.close();
exit(1);
}
while(true)
{
MPI_Irecv(&m, 1, MPI_INT, MPI_ANY_SOURCE, 222, MPI_COMM_WORLD, &requ1);
MPI_Wait(&requ1, &status);
//fout << myid << " recv from " << status.MPI_SOURCE << " m = " << m << "
with tag 222" << endl;
//pthread_mutex_lock(&_dealmutex1);
//cout << "BACKID_REV:" << myid << " recv from " << status.MPI_SOURCE << " m
= " << m << " with tag 222" << endl;
fout << "BACKID_REV:" << myid << " recv from " << status.MPI_SOURCE << " m =
" << m << " with tag 222" << endl;
//fflush(stdout);
fout.flush();
//pthread_mutex_unlock(&_dealmutex1);
//m++;
MPI_Send(&m, 1, MPI_INT, status.MPI_SOURCE, 333, MPI_COMM_WORLD);
//MPI_Isend(&m, 1, MPI_INT, status.MPI_SOURCE, 333, MPI_COMM_WORLD, &requ2);
//pthread_mutex_lock(&_dealmutex1);
//fout << myid << " replies " << status.MPI_SOURCE << " m = " << m << endl;
//cout << "BACKID_SEND:" << myid << " replies " << status.MPI_SOURCE << " m =
" << m << endl;
fout << "BACKID_SEND:" << myid << " replies " << status.MPI_SOURCE << " m = "
<< m << endl;
//fflush(stdout);
fout.flush();
//pthread_mutex_unlock(&_dealmutex);
count++;
//pthread_mutex_unlock(&_dealmutex1);
if(count == 50)
{
fout << "*******backID FINISHED IN " << myid << "********" << endl;
fout.flush();
fout.close();
pthread_exit(NULL);
return 0;
}
};
}
void* backRecv(void* arg)
{
int myid;
pthread_mutex_init(&_dealmutex2, NULL);
stringstream RANK;
MPI_Status status;
MPI_Request requ2;
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
RANK << myid;
cout << myid << " create background message recv" << endl;
int x, y;
//char c;
int m;
int count = 0;
string filename("f_");
filename += RANK.str();
filename += "_backRecv.txt";
fstream fout(filename.c_str(), ios::out);
if(!fout)
{
cout << "can not create the file " << filename << endl;
fout.close();
exit(1);
}
while(true)
{
MPI_Irecv(&m, 1, MPI_INT, MPI_ANY_SOURCE, 333, MPI_COMM_WORLD, &requ2);
MPI_Wait(&requ2, &status);
//pthread_mutex_lock(&_dealmutex2);
fout << "BACKREV:" << myid << " recv from " << status.MPI_SOURCE << " m = "
<< m << " with tag 333" << endl;
fout.flush();
//cout << "BACKREV:" << myid << " recv from " << status.MPI_SOURCE << " m = "
<< m << " with tag 333" << endl;
//fflush(stdout);
//pthread_mutex_unlock(&_dealmutex);
//pthread_mutex_lock(&_dealmutex);
count++;
//pthread_mutex_unlock(&_dealmutex2);
if(count == 50)
{
fout << "*******backRecv FINISHED IN " << myid << "********" << endl;
fout.flush();
fout.close();
pthread_exit(NULL);
return 0;
}
};
}
int main(int argc, char **argv)
{
int myid = 0;
int nprocs = 0;
pthread_t pt1 = 0;
pthread_t pt2 = 0;;
int pret1 = 0;
int pret2 = 0;
int i = 0, j = 0, t = 0;
//MPI_Status status;
MPI_Request requ1;
MPI_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
MPI_Comm_rank(MPI_COMM_WORLD,&myid);
pthread_mutex_init(&_dealmutex, NULL);
for(i=0; i<50; ++i)
{
t = (myid + 1) * i;
MPI_Isend(&myid, 1, MPI_INT, (myid+1)%nprocs, 222, MPI_COMM_WORLD, &requ1);
//MPI_Sendrecv(&t, 1, MPI_INT, (myid+1)%nprocs, 222, &j, 1, MPI_INT,
(myid+1)%nprocs, 333, MPI_COMM_WORLD, &status);
cout << "MAIN:" << myid << " sends to "<< (myid+1)%nprocs << " " << myid <<
endl;
fflush(stdout);
}
pret1 = pthread_create(&pt1, NULL, backRecv, NULL);
if(pret1 != 0)
{
cout << myid << "backRecv Thread Create Failed." << endl;
exit(1);
}
pret2 = pthread_create(&pt2, NULL, backID, NULL);
if(pret2 != 0)
{
cout << myid << "backID Thread Create Failed." << endl;
exit(1);
}
//for(i=0; i<10; ++i)
//{
// c += i;
// MPI_Send(&c, 1, MPI_CHAR, (myid+1)%nprocs, 111, MPI_COMM_WORLD);
// cout << myid << " send " << (char)c << " to " << (myid+1)%nprocs << endl;
//}
pthread_join(pt2, NULL);
cout << "*******************************THREAD 2 SUCESS!" << endl;
pthread_join(pt1, NULL);
cout << "*******************************THREAD 1 SUCESS!" << endl;
MPI_Finalize();
cout << "*******************************MAIN SUCESS!" << endl;
}
This peice of code works fine sometimes. But it can fail too and segmentation
will occur. I will read the reference you point out. Thanks.
List-Post: users@lists.open-mpi.org
Date: Tue, 22 Sep 2009 08:43:11 -0700
From: eugene....@sun.com
To: us...@open-mpi.org
Subject: Re: [OMPI users] How to create multi-thread parallel program using
thread-safe send and recv?
guosong wrote:
Hi all,
I would like to write a multi-thread parallel program. I used pthread. Basicly,
I want to create two background threads besides the main thread(process). For
example, if I use "-np 4", the program should have 4 main processes on four
processors and two background threads for each main process. So there should be
8 threads totally.Wouldn't there be 4 main threads and 8 "slave" threads for a
total of 12 threads? Anyhow, doesn't matter.
I'm not sure where you're starting, but you should at least have a basic
understanding of the different sorts of multithreaded programming models in
MPI. One is that each process is single threaded. Another is the processes
are multithreaded, but only the main thread makes MPI calls. Another is
multithreaded, but only one MPI call at a time. Finally, there can be full
multithreading. You have to decide which of these programming models you want
and which is supported by your MPI (or, if OMPI, how OMPI was built).
For more information, try the MPI_Init_thread() man page or
http://www.mpi-forum.org./docs/mpi21-report.pdf ... see Section 12.4 on "MPI
and Threads".
I wrote a test program and it worked unpredictable. Sometimes I got the result
I want, but sometimes the program got segmentation fault. I used MPI_Isend and
MPI_Irecv for sending and recving. I do not know why? I attached the error
message as follow:
[cheetah:29780] *** Process received signal ***
[cheetah:29780] Signal: Segmentation fault (11)
[cheetah:29780] Signal code: Address not mapped (1)
[cheetah:29780] Failing at address: 0x10
[cheetah:29779] *** Process received signal ***
[cheetah:29779] Signal: Segmentation fault (11)
[cheetah:29779] Signal code: Address not mapped (1)
[cheetah:29779] Failing at address: 0x10
[cheetah:29780] [ 0] /lib64/libpthread.so.0 [0x334b00de70]
[cheetah:29780] [ 1] /act/openmpi/gnu/lib/openmpi/mca_btl_sm.so [0x2b90e1227940]
[cheetah:29780] [ 2] /act/openmpi/gnu/lib/openmpi/mca_pml_ob1.so
[0x2b90e05d61ca]
[cheetah:29780] [ 3] /act/openmpi/gnu/lib/openmpi/mca_pml_ob1.so
[0x2b90e05cac86]
[cheetah:29780] [ 4] /act/openmpi/gnu/lib/libmpi.so.0(PMPI_Send+0x13d)
[0x2b90dde7271d]
[cheetah:29780] [ 5] pt_muti(_Z6backIDPv+0x29b) [0x409929]
[cheetah:29780] [ 6] /lib64/libpthread.so.0 [0x334b0062f7]
[cheetah:29780] [ 7] /lib64/libc.so.6(clone+0x6d) [0x334! a4d1e3d]
[cheetah:29780] *** End of error message ***
[cheetah:29779] [ 0] /lib64/libpthread.so.0 [0x334b00de70]
[cheetah:29779] [ 1] /act/openmpi/gnu/lib/openmpi/mca_btl_sm.so [0x2b39785c0940]
[cheetah:29779] [ 2] /act/openmpi/gnu/lib/openmpi/mca_pml_ob1.so
[0x2b397796f1ca]
[cheetah:29779] [ 3] /act/openmpi/gnu/lib/openmpi/mca_pml_ob1.so
[0x2b3977963c86]
[cheetah:29779] [ 4] /act/openmpi/gnu/lib/libmpi.so.0(PMPI_Send+0x13d)
[0x2b397520b71d]
[cheetah:29779] [ 5] pt_muti(_Z6backIDPv+0x29b) [0x409929]
[cheetah:29779] [ 6] /lib64/libpthread.so.0 [0x334b0062f7]
[cheetah:29779] [ 7] /lib64/libc.so.6(clone+0x6d) [0x334a4d1e3d]
[cheetah:29779] *** End of error message ***
I used gdb to "bt" the error and I got :
Program terminated with signal 11, Segmentation fault.
#0 0x00002b90e1227940 in mca_btl_sm_alloc ()
from /act/openmpi/gnu/lib/openmpi/mca_btl_sm.so
(gdb) bt
#0 0x00002b90e1227940 in mca_btl_sm_alloc ()
from /act/openmpi/gnu/lib/openmpi/mca_btl_sm.so
#1 0x00002b90e05d61ca in mca_pml_ob1_send_request_start_copy ()
from /act/openmpi/gnu/lib/openmpi/mca_pml_ob1.so
#2 0x00002b90e05cac86 in mca_pml_ob1_send ()
from /act/openmpi/gnu/lib/openmpi/mca_pml_ob1.so
#3 0x00002b90dde7271d in PMPI_Send () from /act/openmpi/gnu/lib/libmpi.so.0
#4 0x0000000000409929 in backID (arg=0x0) at pt_muti.cpp:50
#5 0x000000334b0062f7 in start_thread () from /lib64/libpthread.so.0
#6 0x000000334a4d1e3d in clone () from /lib64/libc.so.6
So can anyone give me some suggestions or advice. Thanks very much.
_________________________________________________________________
打工，挣钱，买房子，快来MClub一起”金屋藏娇”！
http://club.msn.cn/?from=10
