ia64/xen-unstable

changeset 19143:86159a906bec

page_alloc: try to put heap control structures on the target node

This also neatly eliminates (in certain cases) the need to reserve a
page between nodes, as the heap control structures will never be freed
and thus can serve as the needed separator.

It also adds an additonal fallback attempting to make use of the
wasted tail of a (group of) page(s) used for the _heap array by putting the
avail array there if it fits.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Jan 30 11:16:52 2009 +0000 (2009-01-30)
parents 9b9078a02f65
children 2e1734aa8db3
files xen/common/page_alloc.c
line diff
     1.1 --- a/xen/common/page_alloc.c	Fri Jan 30 11:14:23 2009 +0000
     1.2 +++ b/xen/common/page_alloc.c	Fri Jan 30 11:16:52 2009 +0000
     1.3 @@ -272,13 +272,16 @@ static unsigned long *avail[MAX_NUMNODES
     1.4  
     1.5  static DEFINE_SPINLOCK(heap_lock);
     1.6  
     1.7 -static void init_node_heap(int node)
     1.8 +static unsigned long init_node_heap(int node, unsigned long mfn,
     1.9 +                                    unsigned long nr)
    1.10  {
    1.11      /* First node to be discovered has its heap metadata statically alloced. */
    1.12      static heap_by_zone_and_order_t _heap_static;
    1.13      static unsigned long avail_static[NR_ZONES];
    1.14      static int first_node_initialised;
    1.15 -
    1.16 +    unsigned long needed = (sizeof(**_heap) +
    1.17 +                            sizeof(**avail) * NR_ZONES +
    1.18 +                            PAGE_SIZE - 1) >> PAGE_SHIFT;
    1.19      int i, j;
    1.20  
    1.21      if ( !first_node_initialised )
    1.22 @@ -286,12 +289,31 @@ static void init_node_heap(int node)
    1.23          _heap[node] = &_heap_static;
    1.24          avail[node] = avail_static;
    1.25          first_node_initialised = 1;
    1.26 +        needed = 0;
    1.27 +    }
    1.28 +#ifdef DIRECTMAP_VIRT_END
    1.29 +    else if ( nr >= needed &&
    1.30 +              mfn + needed <= virt_to_mfn(DIRECTMAP_VIRT_END) )
    1.31 +    {
    1.32 +        _heap[node] = mfn_to_virt(mfn);
    1.33 +        avail[node] = mfn_to_virt(mfn + needed) - sizeof(**avail) * NR_ZONES;
    1.34 +    }
    1.35 +#endif
    1.36 +    else if ( get_order_from_bytes(sizeof(**_heap)) ==
    1.37 +              get_order_from_pages(needed) )
    1.38 +    {
    1.39 +        _heap[node] = alloc_xenheap_pages(get_order_from_pages(needed), 0);
    1.40 +        BUG_ON(!_heap[node]);
    1.41 +        avail[node] = (void *)_heap[node] + (needed << PAGE_SHIFT) -
    1.42 +                      sizeof(**avail) * NR_ZONES;
    1.43 +        needed = 0;
    1.44      }
    1.45      else
    1.46      {
    1.47          _heap[node] = xmalloc(heap_by_zone_and_order_t);
    1.48          avail[node] = xmalloc_array(unsigned long, NR_ZONES);
    1.49          BUG_ON(!_heap[node] || !avail[node]);
    1.50 +        needed = 0;
    1.51      }
    1.52  
    1.53      memset(avail[node], 0, NR_ZONES * sizeof(long));
    1.54 @@ -299,6 +321,8 @@ static void init_node_heap(int node)
    1.55      for ( i = 0; i < NR_ZONES; i++ )
    1.56          for ( j = 0; j <= MAX_ORDER; j++ )
    1.57              INIT_PAGE_LIST_HEAD(&(*_heap[node])[i][j]);
    1.58 +
    1.59 +    return needed;
    1.60  }
    1.61  
    1.62  /* Allocate 2^@order contiguous pages. */
    1.63 @@ -487,12 +511,22 @@ static void init_heap_pages(
    1.64  
    1.65      nid_prev = phys_to_nid(page_to_maddr(pg-1));
    1.66  
    1.67 -    for ( i = 0; i < nr_pages; i++ )
    1.68 +    for ( i = 0; i < nr_pages; nid_prev = nid_curr, i++ )
    1.69      {
    1.70          nid_curr = phys_to_nid(page_to_maddr(pg+i));
    1.71  
    1.72          if ( unlikely(!avail[nid_curr]) )
    1.73 -            init_node_heap(nid_curr);
    1.74 +        {
    1.75 +            unsigned long n;
    1.76 +
    1.77 +            n = init_node_heap(nid_curr, page_to_mfn(pg+i), nr_pages - i);
    1.78 +            if ( n )
    1.79 +            {
    1.80 +                BUG_ON(i + n > nr_pages);
    1.81 +                i += n - 1;
    1.82 +                continue;
    1.83 +            }
    1.84 +        }
    1.85  
    1.86          /*
    1.87           * Free pages of the same node, or if they differ, but are on a
    1.88 @@ -504,8 +538,6 @@ static void init_heap_pages(
    1.89          else
    1.90              printk("Reserving non-aligned node boundary @ mfn %#lx\n",
    1.91                     page_to_mfn(pg+i));
    1.92 -
    1.93 -        nid_prev = nid_curr;
    1.94      }
    1.95  }
    1.96  
    1.97 @@ -533,7 +565,7 @@ static unsigned long avail_heap_pages(
    1.98  #define avail_for_domheap(mfn) !(allocated_in_map(mfn) || is_xen_heap_mfn(mfn))
    1.99  void __init end_boot_allocator(void)
   1.100  {
   1.101 -    unsigned long i;
   1.102 +    unsigned long i, nr = 0;
   1.103      int curr_free, next_free;
   1.104  
   1.105      /* Pages that are free now go to the domain sub-allocator. */
   1.106 @@ -546,8 +578,15 @@ void __init end_boot_allocator(void)
   1.107          if ( next_free )
   1.108              map_alloc(i+1, 1); /* prevent merging in free_heap_pages() */
   1.109          if ( curr_free )
   1.110 -            init_heap_pages(mfn_to_page(i), 1);
   1.111 +            ++nr;
   1.112 +        else if ( nr )
   1.113 +        {
   1.114 +            init_heap_pages(mfn_to_page(i - nr), nr);
   1.115 +            nr = 0;
   1.116 +        }
   1.117      }
   1.118 +    if ( nr )
   1.119 +        init_heap_pages(mfn_to_page(i - nr), nr);
   1.120  
   1.121      if ( !dma_bitsize && (num_online_nodes() > 1) )
   1.122      {