Reduce boot-time memory fragmentation

On certain NUMA configurations having init_node_heap() consume the
first few pages of a new node's memory for internal data structures
leads to unnecessary memory fragmentation, which can - with
sufficiently many nodes - result in there not remaining enough memory
below 4G for Dom0 to set up its swiotlb and PCI-consistent buffers.

Since alloc_boot_pages() generally consumes from the end of available
regions, make init_node_heap() prefer the end of such regions too (so
that fragmentation occurs at only one end of a region).

(Adjustment from first version: Use the tail of the region when the
end addresses alignment is less or equal to the beginning one's, not
just when it's less.)

Further, in order to prefer allocations from higher memory locations,
insert memory regions in reverse order in end_boot_allocator(), with
the exception of inserting one region residing on the boot CPU's node
first (for the statically allocated structures - used for the first
node seen - to be used for this node).

Finally, reduce MAX_ORDER on x86 to the maximum useful value (1Gb), so
that the reservation of a page on node boundaries (again leading to
fragmentation) can be avoided as much as possible (having node
boundaries on less the 1Gb aligned addresses is expected to be rare,
if found in practice at all).

Signed-off-by: Jan Beulich <jbeulich@novell.com>

diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
index 26a0ee9ba1e4fb4ba1f9cca807d4f92b616b60fd..01cd3e119df220d2f287f14b7ac0b06543c94e0e 100644 (file)
--- a/xen/common/page_alloc.c
+++ b/xen/common/page_alloc.c
@@ -231,7 +231,7 @@ static long midsize_alloc_zone_pages;
 static DEFINE_SPINLOCK(heap_lock);
 
 static unsigned long init_node_heap(int node, unsigned long mfn,
-                                    unsigned long nr)
+                                    unsigned long nr, bool_t *use_tail)
 {
     /* First node to be discovered has its heap metadata statically alloced. */
     static heap_by_zone_and_order_t _heap_static;
@@ -250,12 +250,20 @@ static unsigned long init_node_heap(int node, unsigned long mfn,
         needed = 0;
     }
 #ifdef DIRECTMAP_VIRT_END
+    else if ( *use_tail && nr >= needed &&
+              (mfn + nr) <= (virt_to_mfn(DIRECTMAP_VIRT_END - 1) + 1) )
+    {
+        _heap[node] = mfn_to_virt(mfn + nr - needed);
+        avail[node] = mfn_to_virt(mfn + nr - 1) +
+                      PAGE_SIZE - sizeof(**avail) * NR_ZONES;
+    }
     else if ( nr >= needed &&
               (mfn + needed) <= (virt_to_mfn(DIRECTMAP_VIRT_END - 1) + 1) )
     {
         _heap[node] = mfn_to_virt(mfn);
         avail[node] = mfn_to_virt(mfn + needed - 1) +
                       PAGE_SIZE - sizeof(**avail) * NR_ZONES;
+        *use_tail = 0;
     }
 #endif
     else if ( get_order_from_bytes(sizeof(**_heap)) ==
@@ -812,15 +820,24 @@ static void init_heap_pages(
 
         if ( unlikely(!avail[nid_curr]) )
         {
+            unsigned long s = page_to_mfn(pg + i);
+            unsigned long e = page_to_mfn(pg + nr_pages - 1) + 1;
+            bool_t use_tail = (nid_curr == phys_to_nid(pfn_to_paddr(e - 1))) &&
+                              !(s & ((1UL << MAX_ORDER) - 1)) &&
+                              (find_first_set_bit(e) <= find_first_set_bit(s));
             unsigned long n;
 
-            n = init_node_heap(nid_curr, page_to_mfn(pg+i), nr_pages - i);
-            if ( n )
+            n = init_node_heap(nid_curr, page_to_mfn(pg+i), nr_pages - i,
+                               &use_tail);
+            BUG_ON(i + n > nr_pages);
+            if ( n && !use_tail )
             {
-                BUG_ON(i + n > nr_pages);
                 i += n - 1;
                 continue;
             }
+            if ( i + n == nr_pages )
+                break;
+            nr_pages -= n;
         }
 
         /*
@@ -868,6 +885,17 @@ void __init end_boot_allocator(void)
 
     /* Pages that are free now go to the domain sub-allocator. */
     for ( i = 0; i < nr_bootmem_regions; i++ )
+    {
+        struct bootmem_region *r = &bootmem_region_list[i];
+        if ( (r->s < r->e) &&
+             (phys_to_nid(pfn_to_paddr(r->s)) == cpu_to_node(0)) )
+        {
+            init_heap_pages(mfn_to_page(r->s), r->e - r->s);
+            r->e = r->s;
+            break;
+        }
+    }
+    for ( i = nr_bootmem_regions; i-- > 0; )
     {
         struct bootmem_region *r = &bootmem_region_list[i];
         if ( r->s < r->e )

diff --git a/xen/include/asm-x86/config.h b/xen/include/asm-x86/config.h
index 898ad6a1c63db8c1d52c1bd9f9049ba4011c68d4..39475da1d7124bed27d42d532d8fcbc854dd4481 100644 (file)
--- a/xen/include/asm-x86/config.h
+++ b/xen/include/asm-x86/config.h
@@ -26,6 +26,7 @@
 #define CONFIG_NUMA 1
 #define CONFIG_DISCONTIGMEM 1
 #define CONFIG_NUMA_EMU 1
+#define CONFIG_PAGEALLOC_MAX_ORDER (2 * PAGETABLE_ORDER)
 
 /* Intel P4 currently has largest cache line (L2 line size is 128 bytes). */
 #define CONFIG_X86_L1_CACHE_SHIFT 7