ESyS-Particle  4.0.1
comm.hpp
1 
2 // //
3 // Copyright (c) 2003-2011 by The University of Queensland //
4 // Earth Systems Science Computational Centre (ESSCC) //
5 // http://www.uq.edu.au/esscc //
6 // //
7 // Primary Business: Brisbane, Queensland, Australia //
8 // Licensed under the Open Software License version 3.0 //
9 // http://www.opensource.org/licenses/osl-3.0.php //
10 // //
12 
13 
14 //--- point to point communication primitives for TML_Comm ---
15 #include "tml/message/packed_message.h"
16 
17 //--- STL ---
18 #include <map>
19 using std::map;
20 
30 template <typename T>
31 void TML_Comm::send_array(T* data,int ndata,int dest,int tag)
32 {
33  MPI_Send(data,ndata,GetType(*data),dest,tag,m_comm);
34 }
35 
45 template <typename T>
46 void TML_Comm::receive_array(T* data,int ndata,int source,int tag)
47 {
48  MPI_Recv(data,ndata,GetType(*data),source,tag,m_comm,&m_status);
49 }
50 
63 template <typename T,typename P>
64 void TML_Comm::sendrecv_array(T* send_data,int send_count,P* recv_data,int recv_count,int dest,int source,int tag)
65 {
66  MPI_Sendrecv(send_data,send_count,GetType(*send_data),dest,tag,recv_data,recv_count,GetType(*recv_data),source,tag,m_comm,&m_status);
67 }
68 
76 template <typename T>
77 void TML_Comm::send(T data,int dest,int tag)
78 {
79  MPI_Send(&data,1,GetType(data),dest,tag,m_comm);
80 }
81 
82 
91 template <typename T>
92 void TML_Comm::receive(T& data,int source ,int tag)
93 {
94  MPI_Recv(&data,1,GetType(data),source,tag,m_comm,&m_status);
95 }
96 template <typename T,typename P>
107 void TML_Comm::sendrecv(T send_data,P& recv_data,int dest,int source,int tag)
108 {
109  MPI_Sendrecv(&send_data,1,GetType(send_data),dest,tag,&recv_data,1,GetType(recv_data),source,tag,m_comm,&m_status);
110 }
111 
119 template <typename T>
120 void TML_Comm::send_cont(const T& data, int dest, int tag)
121 {
122  int data_size=data.size();
123 
124  // setup buffer
125  typename T::value_type *buffer=new typename T::value_type[data_size];
126 
127  // put data into buffer
128  int count=0;
129 
130  for(typename T::const_iterator iter=data.begin();
131  iter!=data.end();
132  iter++){
133  void* buf=reinterpret_cast<void*>(&(buffer[count])); // get memory adress for buffer element
134  new(buf)(typename T::value_type)(*iter); // initialize object at this adress
135  // the placement new stuff (see Stroustrup, p.255ff) is necessary
136  // because assignment buffer[count]=(*iter) doesn't work if the
137  // T::value_type contains constant member data. Initialization by
138  // calling the copy constructor directly doesn't work for builtin
139  // types
140  count++;
141  }
142 
143  //send size
144  send(data_size,dest,tag);
145 
146  //send data
147  send_array(buffer,data_size,dest,tag+1024);
148 
149  // clean up
150  delete [] buffer;
151 }
152 
162 template <typename T>
163 void TML_Comm::receive_cont(T& data,int source,int tag)
164 {
165  int data_size;
166 
167  //get size
168  receive(data_size,source,tag);
169  // setup recv buffer
170  typename T::value_type *buffer=new typename T::value_type[data_size];
171 
172  //get data
173  receive_array(buffer,data_size,source,tag+1024);
174  // insert into container
175  for(int i=0;i<data_size;i++){
176  data.insert(data.end(),buffer[i]);
177  }
178  delete [] buffer;
179 }
180 
181 
191 template <typename T,typename P>
192 void TML_Comm::sendrecv_cont(T send_data,P& recv_data,int dest,int source,int tag)
193 {
194  int send_count=send_data.size();
195  int recv_count;
196 
197  // setup send buffer
198  typename T::value_type *send_buffer=new typename T::value_type[send_count];
199 
200  // put data into send buffer
201  int count=0;
202  for(typename T::const_iterator iter=send_data.begin();
203  iter!=send_data.end();
204  iter++){
205  void* buf=reinterpret_cast<void*>(&(send_buffer[count])); // get memory adress for buffer element
206  new(buf)(typename T::value_type)(*iter); // initialize object at this adress (see send_cont)
207  count++;
208  }
209 
210  //send/receive size
211  sendrecv(send_count,recv_count,dest,source,tag);
212  // check for recv from Null process -> set recv_count to 0 if so
213  if(source==MPI_PROC_NULL){
214  recv_count=0;
215  }
216 
217  // setup recv buffer
218  typename T::value_type *recv_buffer=new typename T::value_type[recv_count];
219 
220  //send/receive data
221  sendrecv_array(send_buffer,send_count,recv_buffer,recv_count,dest,source,tag+1024);
222 
223  // insert into container
224  for(int i=0;i<recv_count;i++){
225  recv_data.insert(recv_data.end(),recv_buffer[i]);
226  }
227 
228  delete [] send_buffer;
229  delete [] recv_buffer;
230 }
231 
240 template <typename T>
241 void TML_Comm::sendrecv_cont_replace(T& data,int dest,int source,int tag)
242 {
243  int send_count=data.size();
244  int recv_count;
245 
246  // setup send buffer
247  typename T::value_type *send_buffer=new typename T::value_type[send_count];
248 
249  // put data into send buffer
250  int count=0;
251  for(typename T::const_iterator iter=data.begin();
252  iter!=data.end();
253  iter++){
254  void* buf=reinterpret_cast<void*>(&(send_buffer[count])); // get memory adress for buffer element
255  new(buf)(typename T::value_type)(*iter); // initialize object at this adress
256  count++;
257  }
258 
259  //send/receive size
260  sendrecv(send_count,recv_count,dest,source,tag);
261  // check for recv from Null process -> set recv_count to 0 if so
262  if(source==MPI_PROC_NULL){
263  recv_count=0;
264  }
265 
266  // setup recv buffer
267  typename T::value_type *recv_buffer=new typename T::value_type[recv_count];
268 
269  //send/receive data
270  sendrecv_array(send_buffer,send_count,recv_buffer,recv_count,dest,source,tag+1024);
271 
272  // replace data
273  data.erase(data.begin(),data.end());
274  // insert into container
275  for(int i=0;i<recv_count;i++){
276  data.insert(data.end(),recv_buffer[i]);
277  }
278 
279 
280  delete [] send_buffer;
281  delete [] recv_buffer;
282 }
283 
293 template <typename T>
294 void TML_Comm::send_cont_packed(T data, int dest ,bool checked ,int tag)
295 {
296  TML_Packed_Message* msg=new TML_Packed_Message(m_comm);
297  int nb_data=data.size();
298 
299  msg->pack(nb_data); // pack number of items first
300  // pack data
301  for(typename T::const_iterator iter=data.begin();
302  iter!=data.end();
303  iter++){
304  msg->pack(*iter);
305  }
306 
307  //send size
308  send(msg->size(),dest,tag);
309 
310  //send data
311  send_array(msg->buffer(),msg->size(),dest,tag+1024);
312 
313  delete msg;
314 }
315 
325 template <typename T>
326 void TML_Comm::receive_cont_packed(T& data,int source,bool checked,int tag)
327 {
328  int msg_size; // total size of the message
329  int nb_data; // number of data (i.e. the expected data.size())
330 
331  //get size
332  receive(msg_size,source,tag);
333 
334  TML_Packed_Message* msg=new TML_Packed_Message(m_comm,msg_size);
335 
336  //get data
337  receive_array(msg->buffer(),msg_size,source,tag+1024);
338 
339  // extract nuber of items
340  nb_data=msg->pop_int();
341 
342  // unpack data
343  for(int i=0;i<nb_data;i++){
344  typename T::value_type tv;
345  msg->unpack(tv);
346  data.insert(data.end(),tv);
347  }
348  delete msg;
349 }
350 
361 template <typename T,typename P>
362 void TML_Comm::sendrecv_cont_packed(T send_data, P& recv_data, int dest, int source, bool checked,int tag)
363 {
364  TML_Packed_Message* send_msg=new TML_Packed_Message(m_comm);
365 
366  int send_nb_data=send_data.size();
367  int send_msg_size;
368  int recv_nb_data;
369  int recv_msg_size;
370 
371  send_msg->pack(send_nb_data); // pack number of items first
372  // pack data
373  for(typename T::const_iterator iter=send_data.begin();
374  iter!=send_data.end();
375  iter++){
376  send_msg->pack(*iter);
377  }
378  send_msg_size=send_msg->size();
379 
380  //send/receive size
381  sendrecv(send_msg_size,recv_msg_size,dest,source,tag);
382  // check for recv from Null process -> set recv_count to 0 if so
383  if(source==MPI_PROC_NULL){
384  recv_msg_size=0;
385  }
386 
387  // setup receive message
388  TML_Packed_Message* recv_msg=new TML_Packed_Message(m_comm,recv_msg_size);
389 
390  //send/receive data
391  sendrecv_array(send_msg->buffer(),send_msg_size,recv_msg->buffer(),recv_msg_size,dest,source,tag+1024);
392 
393  if(source!=MPI_PROC_NULL){ // if the source exists
394  // extract nuber of items
395  recv_nb_data=recv_msg->pop_int();
396 
397  // unpack data
398  typename T::value_type tv;
399  for(int i=0;i<recv_nb_data;i++){
400  recv_msg->unpack(tv);
401  recv_data.insert(recv_data.end(),tv);
402  }
403  }
404  delete send_msg;
405  delete recv_msg;
406 }
407 
417 template <typename T>
418 void TML_Comm::sendrecv_cont_packed_replace(T& data, int dest, int source, bool checked,int tag)
419 {
420  TML_Packed_Message* send_msg=new TML_Packed_Message(m_comm);
421 
422  int send_nb_data=data.size();
423  int send_msg_size;
424  int recv_nb_data;
425  int recv_msg_size;
426 
427  send_msg->pack(send_nb_data); // pack number of items first
428  // pack data
429  for(typename T::const_iterator iter=data.begin();
430  iter!=data.end();
431  iter++){
432  send_msg->pack(*iter);
433  }
434  send_msg_size=send_msg->size();
435 
436  //send/receive size
437  sendrecv(send_msg_size,recv_msg_size,dest,source,tag);
438  // check for recv from Null process -> set recv_count to 0 if so
439  if(source==MPI_PROC_NULL){
440  recv_msg_size=0;
441  }
442 
443  // setup receive message
444  TML_Packed_Message* recv_msg=new TML_Packed_Message(m_comm,recv_msg_size);
445 
446  //send/receive data
447  sendrecv_array(send_msg->buffer(),send_msg_size,recv_msg->buffer(),recv_msg_size,dest,source,tag+1024);
448 
449  // extract nuber of items
450  recv_nb_data=recv_msg->pop_int();
451 
452  // replace data
453  data.erase(data.begin(),data.end());
454  for(int i=0;i<recv_nb_data;i++){
455  typename T::value_type tv;
456  recv_msg->unpack(tv);
457  data.insert(data.end(),tv);
458  }
459 
460  delete send_msg;
461  delete recv_msg;
462 }
463