linux/NUMA: Work around buggy NUMA node cpusets

If Linux reports multiple nodes with intersecting (or identical) cpusets,
they get merged and the Linux backend asserts false.

Don't assert when insertion returns a merge error anymore.
Just disable NUMA distances if that ever happens since one
object is missing (instead of removing the corresponding row/column).
The BIOS is buggy, the matrix would likely be meaningless anyway.

Thanks to Jeff Becker for reporting the problem on the OMPI list.

By the way, fix the number of nodes returned by look_sysfsnode()
when we failed to read/create some NUMA objects.

diff --git a/src/topology-linux.c b/src/topology-linux.c
index 2841f23a2b0e5716a6337766e2876cc7c89694da..11fd29c7effab32c7e618a280189ca871573bc8c 100644 (file)
--- a/src/topology-linux.c
+++ b/src/topology-linux.c
@@ -1,6 +1,6 @@
 /*
  * Copyright © 2009 CNRS
- * Copyright © 2009-2013 Inria.  All rights reserved.
+ * Copyright © 2009-2014 Inria.  All rights reserved.
  * Copyright © 2009-2013 Université Bordeaux 1
  * Copyright © 2009-2011 Cisco Systems, Inc.  All rights reserved.
  * Copyright © 2010 IBM
@@ -2685,20 +2685,23 @@ look_sysfsnode(struct hwloc_topology *topology,
 
   {
       hwloc_obj_t * nodes = calloc(nbnodes, sizeof(hwloc_obj_t));
-      float * distances = calloc(nbnodes*nbnodes, sizeof(float));
       unsigned *indexes = calloc(nbnodes, sizeof(unsigned));
+      float * distances;
+      int failednodes = 0;
       unsigned index_;
 
-      if (NULL == indexes || NULL == distances || NULL == nodes) {
+      if (NULL == nodes || NULL == indexes) {
           free(nodes);
           free(indexes);
-          free(distances);
           hwloc_bitmap_free(nodeset);
+          nbnodes = 0;
           goto out;
       }
 
-      /* Get node indexes now. We need them in order since Linux groups
-       * sparse distances but keep them in order in the sysfs distance files.
+      /* Unsparsify node indexes.
+       * We'll need them later because Linux groups sparse distances
+       * and keeps them in order in the sysfs distance files.
+       * It'll simplify things in the meantime.
        */
       index_ = 0;
       hwloc_bitmap_foreach_begin (osnode, nodeset) {
@@ -2708,14 +2711,14 @@ look_sysfsnode(struct hwloc_topology *topology,
       hwloc_bitmap_free(nodeset);
 
 #ifdef HWLOC_DEBUG
-      hwloc_debug("%s", "numa distance indexes: ");
+      hwloc_debug("%s", "NUMA indexes: ");
       for (index_ = 0; index_ < nbnodes; index_++) {
        hwloc_debug(" %u", indexes[index_]);
       }
       hwloc_debug("%s", "\n");
 #endif
 
-      /* Get actual distances now */
+      /* Create NUMA objects */
       for (index_ = 0; index_ < nbnodes; index_++) {
           char nodepath[SYSFS_NUMA_NODE_PATH_LEN];
           hwloc_bitmap_t cpuset;
@@ -2725,8 +2728,11 @@ look_sysfsnode(struct hwloc_topology *topology,
 
           sprintf(nodepath, "%s/node%u/cpumap", path, osnode);
           cpuset = hwloc_parse_cpumap(nodepath, data->root_fd);
-          if (!cpuset)
-              continue;
+          if (!cpuset) {
+           /* This NUMA object won't be inserted, we'll ignore distances */
+           failednodes++;
+           continue;
+         }
 
           node = hwloc_alloc_setup_object(HWLOC_OBJ_NODE, osnode);
           node->cpuset = cpuset;
@@ -2738,9 +2744,36 @@ look_sysfsnode(struct hwloc_topology *topology,
           hwloc_debug_1arg_bitmap("os node %u has cpuset %s\n",
                                   osnode, node->cpuset);
           res_obj = hwloc_insert_object_by_cpuset(topology, node);
-          assert(node == res_obj); /* if we got merged, somebody else added NODEs earlier, things went wrong?! */
+         if (node == res_obj) {
+           nodes[index_] = node;
+         } else {
+           /* We got merged somehow, could be a buggy BIOS reporting wrong NUMA node cpuset.
+            * This object disappeared, we'll ignore distances */
+           failednodes++;
+         }
+      }
+
+      if (failednodes) {
+       /* failed to read/create some nodes, don't bother reading/fixing
+        * a distance matrix that would likely be wrong anyway.
+        */
+       nbnodes -= failednodes;
+       distances = NULL;
+      } else {
+       distances = calloc(nbnodes*nbnodes, sizeof(float));
+      }
 
-          nodes[index_] = node;
+      if (NULL == distances) {
+          free(nodes);
+          free(indexes);
+          goto out;
+      }
+
+      /* Get actual distances now */
+      for (index_ = 0; index_ < nbnodes; index_++) {
+          char nodepath[SYSFS_NUMA_NODE_PATH_LEN];
+
+         osnode = indexes[index_];
 
          /* Linux nodeX/distance file contains distance from X to other localities (from ACPI SLIT table or so),
           * store them in slots X*N...X*N+N-1 */