Changeset 2972

Timestamp:

06/26/09 17:22:55 (9 months ago)

Author:

tautges

Message:

Fixing pcomm_unit for more arbitrary cases where you don't know
beforehand how many / which procs you're communicating with.

pcomm_unit: added a test_pack_shared_arbitrary test, to test parallel

read of an arbitrary file (may be commented out eventually)

parallel_unit_tests: replaced ptest.cub with 64bricks_512hex.h5m
ReadParallel?: made some functions public so that they can be called in

tests

MBParallelComm:
- don't initialize size of buffers, leave them to be zero-sized
- move calls to get_buffers into get_interface_procs, controlled by a

boolean flag to the function (normally false, so it isn't done)

- in serial version of exchange_ghost_cells, if buffer is empty, don't

call unpack function; this happens sometimes when a proc receives
entities owned by a third proc, and has to return handles to that
proc; in this case, that third proc didn't send this proc any
entities, even though it's in this proc's communicating procs list;
parallel version doesn't see this 'cuz it never receives a message
with entities from that third proc

- added function to set proc size in addition to proc rank

Location:

MOAB/branches/parallel_ghosting/parallel

Files:

• MBParallelComm.cpp (modified) (8 diffs)
• MBParallelComm.hpp (modified) (3 diffs)
• ReadParallel.hpp (modified) (3 diffs)
• parallel_unit_tests.cpp (modified) (1 diff)
• pcomm_unit.cpp (modified) (4 diffs)

MOAB/branches/parallel_ghosting/parallel/MBParallelComm.cpp

@@ -105 +105 @@
           _new_size = _old_size + (addl_space);    \
       if (_new_size > buff_vec.size()) {               \
-        buff_vec.resize(1.5*_new_size);             \
+        buff_vec.resize(1.5*_new_size);            \
         buff_ptr = &buff_vec[_new_size-(addl_space)];} }
     
@@ -331 +331 @@
     ind = buffProcs.size();
     buffProcs.push_back((unsigned int)to_proc);
-    ownerSBuffs.push_back(std::vector<unsigned char>(INITIAL_BUFF_SIZE));
-    ghostRBuffs.push_back(std::vector<unsigned char>(INITIAL_BUFF_SIZE));
+    ownerSBuffs.push_back(std::vector<unsigned char>());
+    ghostRBuffs.push_back(std::vector<unsigned char>());
       // allocate these other buffs in case we're storing remote handles
-    ownerRBuffs.push_back(std::vector<unsigned char>(INITIAL_BUFF_SIZE));
-    ghostSBuffs.push_back(std::vector<unsigned char>(INITIAL_BUFF_SIZE));
+    ownerRBuffs.push_back(std::vector<unsigned char>());
+    ghostSBuffs.push_back(std::vector<unsigned char>());
     if (is_new) *is_new = true;
   }
@@ -1428 +1428 @@
 {
   if (NULL == ents && 0 == num_ents) {
-    return list_entities(NULL, 0);
+    sharedEnts.print("Shared entities:\n");
+    return MB_SUCCESS;
   }
   
@@ -2612 +2613 @@
     // establish comm procs and buffers for them
   std::set<unsigned int> procs;
-  result = get_interface_procs(procs);
+  result = get_interface_procs(procs, true);
   RRA("Trouble getting iface procs.");
-  for (std::set<unsigned int>::iterator sit = procs.begin(); sit != procs.end(); sit++)
-    get_buffers(*sit);
 
     // resolve shared entity remote handles; implemented in ghost cell exchange
@@ -2719 +2718 @@
       // establish comm procs and buffers for them
     psets.clear();
-    rval = pc[p]->get_interface_procs(psets);
+    rval = pc[p]->get_interface_procs(psets, true);
     if (MB_SUCCESS != rval) return rval;
-    for (std::set<unsigned int>::iterator sit = psets.begin(); sit != psets.end(); sit++)
-      pc[p]->get_buffers(*sit);
-
   }
   
@@ -3168 +3164 @@
   
   //! get processors with which this processor communicates; sets are sorted by processor
-MBErrorCode MBParallelComm::get_interface_procs(std::set<unsigned int> &procs_set)
+MBErrorCode MBParallelComm::get_interface_procs(std::set<unsigned int> &procs_set,
+                                                bool get_buffs)
 {
     // make sure the sharing procs vector is empty
@@ -3201 +3198 @@
     }
   }
+
+  if (get_buffs) {
+    for (std::set<unsigned int>::iterator sit = procs_set.begin(); sit != procs_set.end(); sit++)
+      get_buffers(*sit);
+  }
   
   return MB_SUCCESS;
@@ -3740 +3742 @@
         // incoming ghost entities; unpack; returns entities received
         // both from sending proc and from owning proc (which may be different)
+
+        // buffer could be empty, which means there isn't any message to
+        // unpack (due to this comm proc getting added as a result of indirect
+        // communication); just skip this unpack
+      if (pc->ownerSBuffs[ind].empty()) continue;
+
       unsigned int to_p = pc->buffProcs[ind];
       unsigned char *buff_ptr = &pc->ownerSBuffs[ind][0];

MOAB/branches/parallel_ghosting/parallel/MBParallelComm.hpp

@@ -327 +327 @@
 */
     //! get processors with which this processor shares an interface
-  MBErrorCode get_interface_procs(std::set<unsigned int> &iface_procs);
+  MBErrorCode get_interface_procs(std::set<unsigned int> &iface_procs,
+                                  const bool get_buffs = false);
 
     //! get processors with which this processor communicates
@@ -495 +496 @@
     //! set rank for this pcomm; USED FOR TESTING ONLY!
   void set_rank(unsigned int r);
+  
+    //! set rank for this pcomm; USED FOR TESTING ONLY!
+  void set_size(unsigned int r);
   
     //! get (and possibly allocate) buffers for messages to/from to_proc; returns
@@ -1034 +1038 @@
 }
 
+inline void MBParallelComm::set_size(unsigned int s) 
+{
+  procConfig.proc_size(s);
+}
+
 inline MBErrorCode MBParallelComm::get_sharing_data(MBEntityHandle entity,
                                                     int *ps, 

MOAB/branches/parallel_ghosting/parallel/ReadParallel.hpp

@@ -32 +32 @@
                         const int num_material_sets );
   
-    //! Constructor
-  ReadParallel(MBInterface* impl = NULL, MBParallelComm *pc = NULL);
-
-   //! Destructor
-  virtual ~ReadParallel() {}
-
-  static const char *parallelOptsNames[];
-  
-  enum ParallelOpts {POPT_NONE=0, POPT_BCAST, POPT_BCAST_DELETE, 
-                     POPT_READ_DELETE, POPT_READ_PARALLEL,
-                     POPT_FORMAT, POPT_DEFAULT};
-  
-protected:
-
-private:
   MBErrorCode load_file(const char **file_names,
                         const int num_files,
@@ -65 +50 @@
                         const int bridge_dim,
                         const int num_layers);
+    //! Constructor
+  ReadParallel(MBInterface* impl = NULL, MBParallelComm *pc = NULL);
 
+   //! Destructor
+  virtual ~ReadParallel() {}
+
+  static const char *parallelOptsNames[];
+  
+  enum ParallelOpts {POPT_NONE=0, POPT_BCAST, POPT_BCAST_DELETE, 
+                     POPT_READ_DELETE, POPT_READ_PARALLEL,
+                     POPT_FORMAT, POPT_DEFAULT};
+
+    //! PUBLIC TO ALLOW TESTING
   MBErrorCode delete_nonlocal_entities(std::string &ptag_name,
                                        std::vector<int> &ptag_vals,
@@ -72 +69 @@
   
   MBErrorCode delete_nonlocal_entities(MBEntityHandle file_set);
+
+protected:
+
+private:
 
   MBInterface *mbImpl;

MOAB/branches/parallel_ghosting/parallel/parallel_unit_tests.cpp

@@ -188 +188 @@
   if (!filename) {
 #ifdef SRCDIR
-    filename = STRINGIFY(SRCDIR) "/ptest.cub";
+    filename = STRINGIFY(SRCDIR) "/../test/64bricks_512hex.h5m";
 #else
-    filename = "ptest.cub";
+    filename = "../test/64bricks_512hex.h5m";
 #endif
   }

MOAB/branches/parallel_ghosting/parallel/pcomm_unit.cpp

@@ -4 +4 @@
 #include "MBCore.hpp"
 #include "MeshTopoUtil.hpp"
+#include "ReadParallel.hpp"
+#include "FileOptions.hpp"
 #include "TestUtil.hpp"
 #include <algorithm>
@@ -44 +46 @@
 /** Test pack/unpack of shared entities in 3d*/
 void test_pack_shared_entities_3d();
+/** Test pack/unpack of arbitrary mesh file */
+void test_pack_shared_arbitrary();
 /** Test filter_pstatus function*/
 void test_filter_pstatus();
@@ -70 +74 @@
   num_err += RUN_TEST( test_pack_shared_entities_2d );
   num_err += RUN_TEST( test_pack_shared_entities_3d );
+  num_err += RUN_TEST( test_pack_shared_arbitrary );
   num_err += RUN_TEST( test_filter_pstatus );
   
@@ -1849 +1854 @@
     // now 1 layer of hex ghosts
   rval = MBParallelComm::exchange_ghost_cells(pc, 4, 3, 0, 1, true);
+  CHECK_ERR(rval);
+}
+
+void test_pack_shared_arbitrary()
+{
+#define NP 4
+  MBCore moab[NP];
+  MBParallelComm *pc[NP];
+  for (unsigned int i = 0; i < NP; i++) {
+    pc[i] = new MBParallelComm(&moab[i]);
+    pc[i]->set_rank(i);
+    pc[i]->set_size(NP);
+  }
+
+  std::string ptag_name("MATERIAL_SET");
+  std::vector<int> pa_vec;
+  pa_vec.push_back(0);
+  pa_vec.push_back(4);
+  pa_vec.push_back(2);
+  MBErrorCode rval;
+
+  const char *fnames[] = {"/home/tautges/MOABpar2/test/64bricks_512hex.h5m"};
+  
+  std::string partition_name("MATERIAL_SET");
+  FileOptions fopts(NULL);
+  
+  for (unsigned int i = 0; i < NP; i++) {
+    ReadParallel rp(moab+i, pc[i]);
+    MBEntityHandle tmp_set = 0;
+    std::vector<int> partition_tag_vals;
+    rval = rp.load_file(fnames, 1, tmp_set, ReadParallel::POPT_READ_DELETE,
+                        partition_name, 
+                        partition_tag_vals, true, pa_vec, NULL, 0,
+                        fopts, i, false, -1, -1, -1, -1, 0);
+    CHECK_ERR(rval);
+  }
+  
+  rval = MBParallelComm::resolve_shared_ents(pc, NP, 3);
+  CHECK_ERR(rval);
+
+    // exchange interface cells
+  rval = MBParallelComm::exchange_ghost_cells(pc, NP, -1, -1, 0, true);
+  CHECK_ERR(rval);
+  
+    // now 1 layer of hex ghosts
+  rval = MBParallelComm::exchange_ghost_cells(pc, NP, 3, 0, 1, true);
   CHECK_ERR(rval);
 }