Changeset 5653

Timestamp:

11/01/09 05:08:30 (3 weeks ago)

Author:

balaji

Message:

Update the HYDT_bind_info structure to make it more flexible and
future proof. Also, extend the UI to allow users to make binding
requests more intelligently.

The HYDT_bind_info structure itself should provide enough flexibility
for hwloc (barring any comments from Guillaume). The user still cannot
specify arbitrarily complex binding requests. Not sure how this can be
done without overly complicating the command line interface.

This should fix ticket #911.

Location:

mpich2/trunk/src/pm/hydra

Files:

• include/hydra_tools.h (modified) (1 diff)
• include/hydra_utils.h (modified) (1 diff)
• pm/pmiserv/pmi_proxy_utils.c (modified) (4 diffs)
• tools/bind/bind.c (modified) (10 diffs)
• tools/bind/bind.h (modified) (1 diff)
• tools/bind/hwloc/bind_hwloc.h (modified) (1 diff)
• tools/bind/plpa/bind_plpa.c (modified) (5 diffs)
• tools/bind/plpa/bind_plpa.h (modified) (1 diff)
• ui/mpiexec/utils.c (modified) (1 diff)
• utils/launch/launch.c (modified) (4 diffs)

mpich2/trunk/src/pm/hydra/include/hydra_tools.h

@@ -13 +13 @@
 HYD_status HYDT_bind_init(char *binding, char *bindlib);
 void HYDT_bind_finalize(void);
-HYD_status HYDT_bind_process(int proc_unit_id);
-int HYDT_bind_get_proc_unit_id(int process_id);
+HYD_status HYDT_bind_process(int os_index);
+int HYDT_bind_get_os_index(int process_id);
 
 /* checkpointing */

mpich2/trunk/src/pm/hydra/include/hydra_utils.h

@@ -161 +161 @@
 
 HYD_status HYDU_create_process(char **client_arg, HYD_env_t * env_list,
-                               int *in, int *out, int *err, int *pid, int proc_unit_id);
-HYD_status HYDU_fork_and_exit(int proc_unit_id);
+                               int *in, int *out, int *err, int *pid, int os_index);
+HYD_status HYDU_fork_and_exit(int os_index);
 #if defined HAVE_THREAD_SUPPORT
 HYD_status HYDU_create_thread(void *(*func) (void *), void *args,

mpich2/trunk/src/pm/hydra/pm/pmiserv/pmi_proxy_utils.c

@@ -490 +490 @@
 HYD_status HYD_pmcd_pmi_proxy_launch_procs(void)
 {
-    int i, j, arg, stdin_fd, process_id, proc_unit_id, pmi_id;
+    int i, j, arg, stdin_fd, process_id, os_index, pmi_id;
     char *str, *envstr, *list;
     char *client_args[HYD_NUM_TMP_STRINGS];
@@ -651 +651 @@
             client_args[arg++] = NULL;
 
-            proc_unit_id = HYDT_bind_get_proc_unit_id(process_id);
+            os_index = HYDT_bind_get_os_index(process_id);
             if (pmi_id == 0) {
                 status = HYDU_create_process(client_args, prop_env,
@@ -658 +658 @@
                                              &HYD_pmcd_pmip.downstream.err[process_id],
                                              &HYD_pmcd_pmip.downstream.pid[process_id],
-                                             proc_unit_id);
+                                             os_index);
 
                 HYD_pmcd_pmip.local.stdin_buf_offset = 0;
@@ -676 +676 @@
                                              &HYD_pmcd_pmip.downstream.err[process_id],
                                              &HYD_pmcd_pmip.downstream.pid[process_id],
-                                             proc_unit_id);
+                                             os_index);
             }
             HYDU_ERR_POP(status, "create process returned error\n");

mpich2/trunk/src/pm/hydra/tools/bind/bind.c

@@ -21 +21 @@
 HYD_status HYDT_bind_init(char *binding, char *bindlib)
 {
-    char *bindstr, *bindentry;
-    int sock, core, thread, i, j;
+    char *bindstr, *bindentry, *obj;
+    int i, j, k, use_topo_obj[HYDT_TOPO_END] = { 0 }, child_id, found_obj;
+    HYDT_topo_obj_type_t topo_end;
+    struct HYDT_topo_obj *topo_obj[HYDT_TOPO_END];
     HYD_status status = HYD_SUCCESS;
 
@@ -28 +30 @@
 
     HYDT_bind_info.support_level = HYDT_BIND_NONE;
-    HYDT_bind_info.topology = NULL;
     HYDT_bind_info.bindlib = HYDU_strdup(bindlib);
-
+    HYDT_bind_info.bindmap = NULL;
+
+    /***************************** NONE *****************************/
     if (!binding || !strcmp(binding, "none")) {
         /* If no binding is given, we just set all mappings to -1 */
         HYDU_MALLOC(HYDT_bind_info.bindmap, int *, sizeof(int), status);
-        HYDT_bind_info.num_procs = 1;
+        HYDT_bind_info.total_proc_units = 1;
         HYDT_bind_info.bindmap[0] = -1;
 
@@ -59 +62 @@
     if (HYDT_bind_info.support_level == HYDT_BIND_NONE) {
         HYDU_MALLOC(HYDT_bind_info.bindmap, int *, sizeof(int), status);
-        HYDT_bind_info.num_procs = 1;
+        HYDT_bind_info.total_proc_units = 1;
         HYDT_bind_info.bindmap[0] = -1;
 
@@ -65 +68 @@
     }
 
+
+    /***************************** USER *****************************/
     if (!strncmp(binding, "user:", strlen("user:"))) {
-        /* If the user specified the binding, we don't need to
-         * initialize anything */
-
         /* Find the number of processing elements */
         bindstr = HYDU_strdup(binding + strlen("user:"));
-        HYDT_bind_info.num_procs = 0;
+        HYDT_bind_info.total_proc_units = 0;
         bindentry = strtok(bindstr, ",");
         while (bindentry) {
-            HYDT_bind_info.num_procs++;
+            HYDT_bind_info.total_proc_units++;
             bindentry = strtok(NULL, ",");
         }
 
         /* Find the actual processing elements */
-        HYDU_MALLOC(HYDT_bind_info.bindmap, int *, HYDT_bind_info.num_procs * sizeof(int),
-                    status);
+        HYDU_MALLOC(HYDT_bind_info.bindmap, int *,
+                    HYDT_bind_info.total_proc_units * sizeof(int), status);
         i = 0;
         bindstr = HYDU_strdup(binding + strlen("user:"));
@@ -92 +94 @@
     }
 
-    HYDU_MALLOC(HYDT_bind_info.bindmap, int *, HYDT_bind_info.num_procs * sizeof(int),
-                status);
-
-    for (i = 0; i < HYDT_bind_info.num_procs; i++) {
-
-        /* RR is supported at the basic binding level */
-        if (!strcmp(binding, "rr")) {
+
+    /***************************** RR *****************************/
+    HYDU_MALLOC(HYDT_bind_info.bindmap, int *,
+                HYDT_bind_info.total_proc_units * sizeof(int), status);
+
+    /* RR is supported at the basic binding level */
+    if (!strcmp(binding, "rr")) {
+        for (i = 0; i < HYDT_bind_info.total_proc_units; i++)
             HYDT_bind_info.bindmap[i] = i;
-            continue;
-        }
-
-        /* If we reached here, the user requested for topology aware
-         * binding. */
-        if (HYDT_bind_info.support_level != HYDT_BIND_TOPO)
-            HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
-                                "topology binding not supported on this platform\n");
-
-        if (!strcmp(binding, "buddy")) {
-            thread = i / (HYDT_bind_info.num_sockets * HYDT_bind_info.num_cores);
-
-            core = i % (HYDT_bind_info.num_sockets * HYDT_bind_info.num_cores);
-            core /= HYDT_bind_info.num_sockets;
-
-            sock = i % HYDT_bind_info.num_sockets;
-        }
-        else if (!strcmp(binding, "pack")) {
-            sock = i / (HYDT_bind_info.num_cores * HYDT_bind_info.num_threads);
-
-            core = i % (HYDT_bind_info.num_cores * HYDT_bind_info.num_threads);
-            core /= HYDT_bind_info.num_threads;
-
-            thread = i % HYDT_bind_info.num_threads;
+
+        goto fn_exit;
+    }
+
+
+    /* If we reached here, the user requested for topology aware
+     * binding. */
+    if (HYDT_bind_info.support_level != HYDT_BIND_TOPO)
+        HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
+                            "topology binding not supported on this platform\n");
+
+
+    /***************************** TOPO *****************************/
+    if (!strncmp(binding, "topo", strlen("topo"))) {
+        bindstr = HYDU_strdup(binding);
+        bindentry = strtok(bindstr, ":");
+        bindentry = strtok(NULL, ":");
+
+        if (bindentry == NULL) {
+            /* No extension option specified; use all resources */
+            for (i = HYDT_TOPO_MACHINE; i < HYDT_TOPO_END; i++)
+                use_topo_obj[i] = 1;
         }
         else {
-            HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR, "unknown binding option\n");
-        }
-
-        for (j = 0; j < HYDT_bind_info.num_procs; j++) {
-            if (HYDT_bind_info.topology[j].socket_rank == sock &&
-                HYDT_bind_info.topology[j].core_rank == core &&
-                HYDT_bind_info.topology[j].thread_rank == thread) {
-                HYDT_bind_info.bindmap[i] = HYDT_bind_info.topology[j].processor_id;
+            obj = strtok(bindentry, ",");
+            do {
+                if (!strcmp(obj, "node"))
+                    use_topo_obj[HYDT_TOPO_NODE] = 1;
+                else if (!strcmp(obj, "socket") || !strcmp(obj, "sockets"))
+                    use_topo_obj[HYDT_TOPO_SOCKET] = 1;
+                else if (!strcmp(obj, "core") || !strcmp(obj, "cores"))
+                    use_topo_obj[HYDT_TOPO_CORE] = 1;
+                else if (!strcmp(obj, "thread") || !strcmp(obj, "threads"))
+                    use_topo_obj[HYDT_TOPO_THREAD] = 1;
+                else
+                    HYDU_ERR_POP(status, "unrecognized binding option\n");
+
+                obj = strtok(NULL, ",");
+            } while (obj);
+        }
+
+        for (i = HYDT_TOPO_END - 1; i > HYDT_TOPO_MACHINE; i--) {
+            if (use_topo_obj[i])
+                use_topo_obj[i - 1] = 1;
+        }
+
+        topo_end = HYDT_TOPO_END;
+        for (i = HYDT_TOPO_MACHINE; i < HYDT_TOPO_END; i++) {
+            if (use_topo_obj[i] == 0) {
+                topo_end = i;
                 break;
             }
         }
-    }
+
+        /* Initialize indices and topology objects */
+        topo_obj[HYDT_TOPO_MACHINE] = &HYDT_bind_info.machine;
+        for (j = HYDT_TOPO_MACHINE; j < HYDT_TOPO_END; j++) {
+            if (j)
+                topo_obj[j] = topo_obj[j - 1]->children;
+        }
+
+        for (i = 0; i < HYDT_bind_info.total_proc_units; i++) {
+            HYDT_bind_info.bindmap[i] = topo_obj[HYDT_TOPO_END - 1]->os_index;
+
+            /* If we are done, break out */
+            if (i == HYDT_bind_info.total_proc_units - 1)
+                break;
+
+            /* If not, increment the object structure */
+            found_obj = 0;
+
+            for (j = HYDT_TOPO_END - 1; j > HYDT_TOPO_MACHINE; j--) {
+
+                /* If our topology depth is greater than what the user
+                 * requested, don't try to find any more siblings */
+                if (j >= topo_end)
+                    continue;
+
+                child_id = HYDT_TOPO_CHILD_ID(topo_obj[j]);
+                if (child_id < topo_obj[j]->parent->num_children - 1) {
+                    /* This object is not the last of the siblings;
+                     * move to the next sibling */
+                    topo_obj[j] = &topo_obj[j]->parent->children[child_id + 1];
+                    for (k = j + 1; k < HYDT_TOPO_END; k++)
+                        topo_obj[k] = topo_obj[k - 1]->children;
+
+                    found_obj = 1;
+                    break;
+                }
+            }
+
+            if (!found_obj) {
+                /* Initialize indices and topology objects */
+                topo_obj[HYDT_TOPO_MACHINE] = &HYDT_bind_info.machine;
+                for (j = HYDT_TOPO_MACHINE; j < HYDT_TOPO_END; j++) {
+                    if (j)
+                        topo_obj[j] = topo_obj[j - 1]->children;
+                }
+            }
+        }
+
+        goto fn_exit;
+    }
+
+
+    /* If we reached here, the user requested for topology aware
+     * binding. */
+    if (HYDT_bind_info.support_level != HYDT_BIND_MEMTOPO)
+        HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
+                            "memory topology binding not supported on this platform\n");
+
+    /***************************** TOPOMEM *****************************/
+    if (!strncmp(binding, "topomem", strlen("topomem"))) {
+        HYDU_ERR_SETANDJUMP(status, HYD_INTERNAL_ERROR,
+                            "memory-aware topology binding is not implemented yet\n");
+    }
+
 
   fn_exit:
@@ -147 +229 @@
 }
 
+static void cleanup_topo_level(struct HYDT_topo_obj level)
+{
+    int i;
+
+    level.parent = NULL;
+
+    if (level.shared_memory_depth)
+        HYDU_FREE(level.shared_memory_depth);
+
+    if (level.children)
+        for (i = 0; i < level.num_children; i++)
+            cleanup_topo_level(level.children[i]);
+}
+
 void HYDT_bind_finalize(void)
 {
@@ -155 +251 @@
         HYDU_FREE(HYDT_bind_info.bindlib);
 
-    if (HYDT_bind_info.topology)
-        HYDU_FREE(HYDT_bind_info.topology);
-}
-
-HYD_status HYDT_bind_process(int core)
+    cleanup_topo_level(HYDT_bind_info.machine);
+}
+
+HYD_status HYDT_bind_process(int os_index)
 {
     HYD_status status = HYD_SUCCESS;
@@ -167 +262 @@
 #if defined HAVE_PLPA
     if (!strcmp(HYDT_bind_info.bindlib, "plpa")) {
-        status = HYDT_bind_plpa_process(core);
+        status = HYDT_bind_plpa_process(os_index);
         HYDU_ERR_POP(status, "PLPA failure binding process to core\n");
     }
@@ -174 +269 @@
 #if defined HAVE_HWLOC
     if (!strcmp(HYDT_bind_info.bindlib, "hwloc")) {
-        status = HYDT_bind_hwloc_process(core);
+        status = HYDT_bind_hwloc_process(os_index);
         HYDU_ERR_POP(status, "HWLOC failure binding process to core\n");
     }
@@ -187 +282 @@
 }
 
-
-int HYDT_bind_get_proc_unit_id(int id)
-{
-    return HYDT_bind_info.bindmap[id % HYDT_bind_info.num_procs];
-}
+int HYDT_bind_get_os_index(int process_id)
+{
+    /* TODO: Allow the binding layer to export CPU sets instead of
+     * single units */
+
+    return HYDT_bind_info.bindmap[process_id % HYDT_bind_info.total_proc_units];
+}

mpich2/trunk/src/pm/hydra/tools/bind/bind.h

@@ -10 +10 @@
 #include "hydra_utils.h"
 
+#define HYDT_TOPO_CHILD_ID(obj) \
+    ((((char *) obj) - ((char *) obj->parent->children)) / sizeof(struct HYDT_topo_obj))
+
 typedef enum {
     HYDT_BIND_NONE = 0,
     HYDT_BIND_BASIC,
-    HYDT_BIND_TOPO
+    HYDT_BIND_TOPO,
+    HYDT_BIND_MEMTOPO
 } HYDT_bind_support_level_t;
+
+typedef enum {
+    HYDT_TOPO_MACHINE = 0, /* Cache-coherent set of processors */
+    HYDT_TOPO_NODE, /* Sockets sharing memory dimms */
+    HYDT_TOPO_SOCKET,
+    HYDT_TOPO_CORE,
+    HYDT_TOPO_THREAD,
+    HYDT_TOPO_END /* The last element */
+} HYDT_topo_obj_type_t;
+
+struct HYDT_topo_obj {
+    HYDT_topo_obj_type_t type;
+
+    int os_index; /* OS index */
+
+    struct HYDT_topo_obj *parent;
+
+    int num_children;
+    struct HYDT_topo_obj *children;
+
+    /* Depth of the shared memory regions. This is a pointer to
+     * accomodate multiple levels of memory shared by this set of
+     * processing units. */
+    int *shared_memory_depth;
+};
 
 struct HYDT_bind_info {
     HYDT_bind_support_level_t support_level;
 
-    int num_procs;
-    int num_sockets;
-    int num_cores;
-    int num_threads;
+    char *bindlib;
+    int *bindmap;
 
-    int *bindmap;
-    char *bindlib;
+    /* This is needed for all binding levels, except "NONE" */
+    int total_proc_units;
 
-    struct HYDT_topology {
-        int processor_id;
-
-        int socket_rank;
-        int socket_id;
-
-        int core_rank;
-        int core_id;
-
-        int thread_rank;
-        int thread_id;
-    } *topology;
+    struct HYDT_topo_obj machine;
 };
 

mpich2/trunk/src/pm/hydra/tools/bind/hwloc/bind_hwloc.h

@@ -12 +12 @@
 
 HYD_status HYDT_bind_hwloc_init(HYDT_bind_support_level_t * support_level);
-HYD_status HYDT_bind_hwloc_process(int core);
+HYD_status HYDT_bind_hwloc_process(int os_index);
 
 #endif /* BIND_PLPA_H_INCLUDED */

mpich2/trunk/src/pm/hydra/tools/bind/plpa/bind_plpa.c

@@ -17 +17 @@
 {
     PLPA_NAME(api_type_t) p;
-    int ret, i, j, max, id;
-    int processor, sock, core, thread;
+    int ret, i, j, k, proc_id, socket_id, core_id, max, total_cores;
+    struct HYDT_topo_obj *node, *sock, *core, *thread;
     HYD_status status = HYD_SUCCESS;
 
@@ -31 +31 @@
     /* Find the maximum number of processing elements */
     ret = PLPA_NAME(get_processor_data) (PLPA_NAME_CAPS(COUNT_ONLINE),
-                                         &HYDT_bind_info.num_procs, &max);
+                                         &HYDT_bind_info.total_proc_units, &max);
     if (ret) {
         /* Unable to get number of processors */
@@ -38 +38 @@
     }
 
-    HYDU_MALLOC(HYDT_bind_info.topology, struct HYDT_topology *,
-                HYDT_bind_info.num_procs * sizeof(struct HYDT_topology), status);
-    for (i = 0; i < HYDT_bind_info.num_procs; i++) {
-        HYDT_bind_info.topology[i].processor_id = -1;
-        HYDT_bind_info.topology[i].socket_rank = -1;
-        HYDT_bind_info.topology[i].socket_id = -1;
-        HYDT_bind_info.topology[i].core_rank = -1;
-        HYDT_bind_info.topology[i].core_id = -1;
-        HYDT_bind_info.topology[i].thread_rank = -1;
-        HYDT_bind_info.topology[i].thread_id = -1;
-    }
-
-    for (i = 0; i < HYDT_bind_info.num_procs; i++) {
-        ret = PLPA_NAME(get_processor_id) (i, PLPA_NAME_CAPS(COUNT_ALL), &processor);
-        if (ret) {
-            /* Unable to get processor ID */
-            HYDU_warn_printf("plpa get processor id failed\n");
-            if (HYDT_bind_info.topology)
-                HYDU_FREE(HYDT_bind_info.topology);
-            goto fn_fail;
-        }
-        HYDT_bind_info.topology[i].processor_id = processor;
-    }
-
     /* We have qualified for basic binding support level */
     *support_level = HYDT_BIND_BASIC;
 
-    /* PLPA only gives information about sockets and cores */
-    ret = PLPA_NAME(get_socket_info) (&HYDT_bind_info.num_sockets, &max);
+    /* Setup the machine level */
+    HYDT_bind_info.machine.type = HYDT_TOPO_MACHINE;
+    HYDT_bind_info.machine.os_index = -1; /* This is a set, not a single unit */
+    HYDT_bind_info.machine.parent = NULL;
+    HYDT_bind_info.machine.num_children = 1;
+    HYDU_MALLOC(HYDT_bind_info.machine.children, struct HYDT_topo_obj *,
+                sizeof(struct HYDT_topo_obj), status);
+    HYDT_bind_info.machine.shared_memory_depth = NULL;
+
+    /* Setup the node level */
+    node = &HYDT_bind_info.machine.children[0];
+    node->type = HYDT_TOPO_NODE;
+    node->os_index = -1;
+    node->parent =  &HYDT_bind_info.machine;
+    ret = PLPA_NAME(get_socket_info) (&node->num_children, &max);
     if (ret) {
-        /* Unable to get number of sockets */
         HYDU_warn_printf("plpa get socket info failed\n");
         goto fn_fail;
     }
+    HYDU_MALLOC(node->children, struct HYDT_topo_obj *,
+                sizeof(struct HYDT_topo_obj) * node->num_children, status);
+    node->shared_memory_depth = NULL;
 
-    ret = PLPA_NAME(get_core_info) (0, &HYDT_bind_info.num_cores, &max);
-    if (ret) {
-        /* Unable to get number of cores */
-        HYDU_warn_printf("plpa get core info failed\n");
+    /* Setup the socket level */
+    total_cores = 0;
+    for (i = 0; i < node->num_children; i++) {
+        sock = &node->children[i];
+        sock->type = HYDT_TOPO_SOCKET;
+        sock->os_index = -1;
+        sock->parent = node;
+
+        ret = PLPA_NAME(get_socket_id) (i, &socket_id);
+        if (ret) {
+            HYDU_warn_printf("plpa get socket id failed\n");
+            goto fn_fail;
+        }
+
+        ret = PLPA_NAME(get_core_info) (socket_id, &sock->num_children, &max);
+        if (ret) {
+            HYDU_warn_printf("plpa get core info failed\n");
+            goto fn_fail;
+        }
+        HYDU_MALLOC(sock->children, struct HYDT_topo_obj *,
+                    sizeof(struct HYDT_topo_obj) * sock->num_children, status);
+        sock->shared_memory_depth = NULL;
+
+        total_cores += sock->num_children;
+    }
+
+    if (HYDT_bind_info.total_proc_units % total_cores) {
+        HYDU_warn_printf("total processing units is not a multiple of total cores\n");
         goto fn_fail;
     }
 
-    HYDT_bind_info.num_threads = HYDT_bind_info.num_procs /
-        (HYDT_bind_info.num_sockets * HYDT_bind_info.num_cores);
+    /* Core level objects */
+    for (i = 0; i < node->num_children; i++) {
+        sock = &node->children[i];
 
-    /* Find the socket and core IDs for all processor IDs */
-    for (i = 0; i < HYDT_bind_info.num_procs; i++) {
-        ret = PLPA_NAME(map_to_socket_core) (HYDT_bind_info.topology[i].processor_id,
-                                             &sock, &core);
+        for (j = 0; j < sock->num_children; j++) {
+            core = &sock->children[j];
+            core->type = HYDT_TOPO_CORE;
+            core->os_index = -1;
+            core->parent = sock;
+            core->num_children = HYDT_bind_info.total_proc_units / total_cores;
+            HYDU_MALLOC(core->children, struct HYDT_topo_obj *,
+                        sizeof(struct HYDT_topo_obj) * sock->num_children, status);
+            core->shared_memory_depth = NULL;
+
+            for (k = 0; k < core->num_children; k++) {
+                thread = &core->children[k];
+                thread->type = HYDT_TOPO_THREAD;
+                thread->os_index = -1;
+                thread->parent = core;
+                thread->num_children = 0;
+                thread->children = NULL;
+                thread->shared_memory_depth = NULL;
+            }
+        }
+    }
+
+    for (i = 0; i < HYDT_bind_info.total_proc_units; i++) {
+        ret = PLPA_NAME(get_processor_id) (i, PLPA_NAME_CAPS(COUNT_ONLINE), &proc_id);
         if (ret) {
-            /* Unable to get number of cores */
+            HYDU_warn_printf("plpa unable to get processor id\n");
+            goto fn_fail;
+        }
+
+        ret = PLPA_NAME(map_to_socket_core) (proc_id, &socket_id, &core_id);
+        if (ret) {
             HYDU_warn_printf("plpa unable to map socket to core\n");
             goto fn_fail;
         }
 
-        HYDT_bind_info.topology[i].socket_id = sock;
-        HYDT_bind_info.topology[i].core_id = core;
-
-        thread = -1;
-        for (j = 0; j < i; j++)
-            if (HYDT_bind_info.topology[j].socket_id == sock &&
-                HYDT_bind_info.topology[j].core_id == core)
-                thread = HYDT_bind_info.topology[j].thread_id;
-        thread++;
-
-        HYDT_bind_info.topology[i].thread_id = thread;
-        HYDT_bind_info.topology[i].thread_rank = thread;
-    }
-
-    /* Find the rank of each socket ID */
-    for (i = 0; i < HYDT_bind_info.num_sockets; i++) {
-        ret = PLPA_NAME(get_socket_id) (i, &id);
-        if (ret) {
-            /* Unable to get socket id */
-            HYDU_warn_printf("plpa unable to get socket id\n");
-            goto fn_fail;
+        for (j = 0; j < HYDT_bind_info.total_proc_units / total_cores; j++) {
+            thread = &node->children[socket_id].children[core_id].children[j];
+            if (thread->os_index == -1) {
+                thread->os_index = i;
+                break;
+            }
         }
-        for (j = 0; j < HYDT_bind_info.num_procs; j++)
-            if (HYDT_bind_info.topology[j].socket_id == id)
-                HYDT_bind_info.topology[j].socket_rank = i;
-    }
-
-    /* Find the rank of each core ID */
-    for (i = 0; i < HYDT_bind_info.num_cores; i++) {
-        ret = PLPA_NAME(get_core_id) (HYDT_bind_info.topology[0].socket_id, i, &id);
-        if (ret) {
-            /* Unable to get socket id */
-            HYDU_warn_printf("plpa unable to get socket id\n");
-            goto fn_fail;
-        }
-        for (j = 0; j < HYDT_bind_info.num_procs; j++)
-            if (HYDT_bind_info.topology[j].core_id == id)
-                HYDT_bind_info.topology[j].core_rank = i;
     }
 
@@ -144 +154 @@
 }
 
-HYD_status HYDT_bind_plpa_process(int core)
+HYD_status HYDT_bind_plpa_process(int os_index)
 {
     int ret;
@@ -152 +162 @@
     HYDU_FUNC_ENTER();
 
-    /* If the specified core is negative, we just ignore it */
-    if (core < 0)
+    /* If the specified os_index is negative, we just ignore it */
+    if (os_index < 0)
         goto fn_exit;
 
     PLPA_NAME_CAPS(CPU_ZERO) (&cpuset);
-    PLPA_NAME_CAPS(CPU_SET) (core % HYDT_bind_info.num_procs, &cpuset);
+    PLPA_NAME_CAPS(CPU_SET) (os_index % HYDT_bind_info.total_proc_units, &cpuset);
     ret = PLPA_NAME(sched_setaffinity) (0, 1, &cpuset);
     if (ret)

mpich2/trunk/src/pm/hydra/tools/bind/plpa/bind_plpa.h

@@ -9 +9 @@
 
 HYD_status HYDT_bind_plpa_init(HYDT_bind_support_level_t * support_level);
-HYD_status HYDT_bind_plpa_process(int core);
+HYD_status HYDT_bind_plpa_process(int os_index);
 
 #endif /* BIND_PLPA_H_INCLUDED */

mpich2/trunk/src/pm/hydra/ui/mpiexec/utils.c

@@ -685 +685 @@
         printf("        none -- no binding\n");
         printf("        rr   -- round-robin as OS assigned processor IDs\n");
-        printf("        buddy -- order of least shared resources\n");
-        printf("        pack  -- order of most shared resources\n\n");
         printf("        user:0,1,3,2 -- user specified binding\n");
+        printf("        topo -- CPU topology-aware binding\n");
+        printf("        topomem -- memory topology-aware binding\n\n");
+
+        printf("    CPU options (supported on topology-capable binding libs):\n");
+        printf("        topo (with no options) -- use all CPU resources\n");
+        printf("        topo:sockets,cores,threads -- use all CPU resources\n");
+        printf("        topo:sockets,cores -- avoid using multiple threads on a core\n");
+        printf("        topo:sockets -- avoid using multiple cores on a socket\n");
+
+        printf("    Memory options (supported on memory topology aware binding libs):\n");
+        printf("        topomem:l1,l2,l3,mem -- use all memory resources\n");
+        printf("        topomem:l2,l3,mem -- avoid sharing l1 cache\n");
+        printf("        topomem:l3,mem -- avoid using l2 cache\n");
+        printf("        topomem:mem -- avoid using l3 cache\n");
         HYDU_ERR_SETANDJUMP(status, HYD_GRACEFUL_ABORT, "");
     }

mpich2/trunk/src/pm/hydra/utils/launch/launch.c

@@ -9 +9 @@
 
 HYD_status HYDU_create_process(char **client_arg, HYD_env_t * env_list,
-                               int *in, int *out, int *err, int *pid, int proc_unit_id)
+                               int *in, int *out, int *err, int *pid, int os_index)
 {
     int inpipe[2], outpipe[2], errpipe[2], tpid;
@@ -60 +60 @@
         }
 
-        if (proc_unit_id >= 0) {
-            status = HYDT_bind_process(proc_unit_id);
+        if (os_index >= 0) {
+            status = HYDT_bind_process(os_index);
             HYDU_ERR_POP(status, "bind process failed\n");
         }
@@ -101 +101 @@
 
 
-HYD_status HYDU_fork_and_exit(int proc_unit_id)
+HYD_status HYDU_fork_and_exit(int os_index)
 {
     pid_t tpid;
@@ -115 +115 @@
         close(2);
 
-        if (proc_unit_id >= 0) {
-            status = HYDT_bind_process(proc_unit_id);
+        if (os_index >= 0) {
+            status = HYDT_bind_process(os_index);
             HYDU_ERR_POP(status, "bind process failed\n");
         }