ia64/xen-unstable

changeset 14106:8afe591c272b

xen memory allocator: Dynamically allocate per-numa-node metadata.
Signed-off-by: Keir Fraser <keir@xensource.com>
author Keir Fraser <keir@xensource.com>
date Sat Feb 24 12:19:22 2007 +0000 (2007-02-24)
parents e9bd3267ff23
children 1e5a83fb928b
files xen/common/page_alloc.c
line diff
     1.1 --- a/xen/common/page_alloc.c	Fri Feb 23 17:26:07 2007 -0800
     1.2 +++ b/xen/common/page_alloc.c	Sat Feb 24 12:19:22 2007 +0000
     1.3 @@ -319,12 +319,24 @@ unsigned long alloc_boot_pages(
     1.4  
     1.5  #define pfn_dom_zone_type(_pfn) (fls(_pfn) - 1)
     1.6  
     1.7 -static struct list_head heap[NR_ZONES][MAX_NUMNODES][MAX_ORDER+1];
     1.8 +typedef struct list_head heap_by_zone_and_order_t[NR_ZONES][MAX_ORDER+1];
     1.9 +static heap_by_zone_and_order_t _heap0;
    1.10 +static heap_by_zone_and_order_t *_heap[MAX_NUMNODES];
    1.11 +#define heap(node, zone, order) ((*_heap[node])[zone][order])
    1.12  
    1.13 -static unsigned long avail[NR_ZONES][MAX_NUMNODES];
    1.14 +static unsigned long avail0[NR_ZONES];
    1.15 +static unsigned long *avail[MAX_NUMNODES];
    1.16  
    1.17  static DEFINE_SPINLOCK(heap_lock);
    1.18  
    1.19 +static void init_heap_block(heap_by_zone_and_order_t *heap_block)
    1.20 +{
    1.21 +    int i, j;
    1.22 +    for ( i = 0; i < NR_ZONES; i++ )
    1.23 +        for ( j = 0; j <= MAX_ORDER; j++ )
    1.24 +            INIT_LIST_HEAD(&(*heap_block)[i][j]);
    1.25 +}
    1.26 +
    1.27  /* Allocate 2^@order contiguous pages. */
    1.28  static struct page_info *alloc_heap_pages(
    1.29      unsigned int zone_lo, unsigned zone_hi,
    1.30 @@ -353,17 +365,18 @@ static struct page_info *alloc_heap_page
    1.31          for ( zone = zone_hi; zone >= zone_lo; --zone )
    1.32          {
    1.33              /* check if target node can support the allocation */
    1.34 -            if ( avail[zone][node] >= request )
    1.35 +            if ( avail[node] && (avail[node][zone] >= request) )
    1.36              {
    1.37                  /* Find smallest order which can satisfy the request. */
    1.38                  for ( j = order; j <= MAX_ORDER; j++ )
    1.39                  {
    1.40 -                    if ( !list_empty(&heap[zone][node][j]) )
    1.41 +                    if ( !list_empty(&heap(node, zone, j)) )
    1.42                          goto found;
    1.43                  }
    1.44              }
    1.45          }
    1.46 -        /* pick next node, wrapping around if needed */
    1.47 +
    1.48 +        /* Pick next node, wrapping around if needed. */
    1.49          if ( ++node == num_nodes )
    1.50              node = 0;
    1.51      }
    1.52 @@ -373,20 +386,20 @@ static struct page_info *alloc_heap_page
    1.53      return NULL;
    1.54  
    1.55   found: 
    1.56 -    pg = list_entry(heap[zone][node][j].next, struct page_info, list);
    1.57 +    pg = list_entry(heap(node, zone, j).next, struct page_info, list);
    1.58      list_del(&pg->list);
    1.59  
    1.60      /* We may have to halve the chunk a number of times. */
    1.61      while ( j != order )
    1.62      {
    1.63          PFN_ORDER(pg) = --j;
    1.64 -        list_add_tail(&pg->list, &heap[zone][node][j]);
    1.65 +        list_add_tail(&pg->list, &heap(node, zone, j));
    1.66          pg += 1 << j;
    1.67      }
    1.68      
    1.69      map_alloc(page_to_mfn(pg), request);
    1.70 -    ASSERT(avail[zone][node] >= request);
    1.71 -    avail[zone][node] -= request;
    1.72 +    ASSERT(avail[node][zone] >= request);
    1.73 +    avail[node][zone] -= request;
    1.74  
    1.75      spin_unlock(&heap_lock);
    1.76  
    1.77 @@ -408,8 +421,8 @@ static void free_heap_pages(
    1.78      spin_lock(&heap_lock);
    1.79  
    1.80      map_free(page_to_mfn(pg), 1 << order);
    1.81 -    avail[zone][node] += 1 << order;
    1.82 -    
    1.83 +    avail[node][zone] += 1 << order;
    1.84 +
    1.85      /* Merge chunks as far as possible. */
    1.86      while ( order < MAX_ORDER )
    1.87      {
    1.88 @@ -435,12 +448,12 @@ static void free_heap_pages(
    1.89          
    1.90          order++;
    1.91  
    1.92 -        /* after merging, pg should be in the same node */
    1.93 -        ASSERT(phys_to_nid(page_to_maddr(pg)) == node );
    1.94 +        /* After merging, pg should remain in the same node. */
    1.95 +        ASSERT(phys_to_nid(page_to_maddr(pg)) == node);
    1.96      }
    1.97  
    1.98      PFN_ORDER(pg) = order;
    1.99 -    list_add_tail(&pg->list, &heap[zone][node][order]);
   1.100 +    list_add_tail(&pg->list, &heap(node, zone, order));
   1.101  
   1.102      spin_unlock(&heap_lock);
   1.103  }
   1.104 @@ -469,6 +482,14 @@ void init_heap_pages(
   1.105      {
   1.106          nid_curr = phys_to_nid(page_to_maddr(pg+i));
   1.107  
   1.108 +        if ( !avail[nid_curr] )
   1.109 +        {
   1.110 +            avail[nid_curr] = xmalloc_array(unsigned long, NR_ZONES);
   1.111 +            memset(avail[nid_curr], 0, NR_ZONES * sizeof(long));
   1.112 +            _heap[nid_curr] = xmalloc(heap_by_zone_and_order_t);
   1.113 +            init_heap_block(_heap[nid_curr]);
   1.114 +        }
   1.115 +
   1.116          /*
   1.117           * free pages of the same node, or if they differ, but are on a
   1.118           * MAX_ORDER alignement boundary (which already get reserved)
   1.119 @@ -492,25 +513,27 @@ static unsigned long avail_heap_pages(
   1.120  
   1.121      if ( zone_hi >= NR_ZONES )
   1.122          zone_hi = NR_ZONES - 1;
   1.123 -    for ( zone = zone_lo; zone <= zone_hi; zone++ )
   1.124 -        for ( i = 0; i < num_nodes; i++ )
   1.125 +
   1.126 +    for ( i = 0; i < num_nodes; i++ )
   1.127 +    {
   1.128 +        if ( !avail[i] )
   1.129 +            continue;
   1.130 +        for ( zone = zone_lo; zone <= zone_hi; zone++ )
   1.131              if ( (node == -1) || (node == i) )
   1.132 -                free_pages += avail[zone][i];
   1.133 +                free_pages += avail[i][zone];
   1.134 +    }
   1.135  
   1.136      return free_pages;
   1.137  }
   1.138  
   1.139  void end_boot_allocator(void)
   1.140  {
   1.141 -    unsigned long i, j, k;
   1.142 +    unsigned long i;
   1.143      int curr_free, next_free;
   1.144  
   1.145 -    memset(avail, 0, sizeof(avail));
   1.146 -
   1.147 -    for ( i = 0; i < NR_ZONES; i++ )
   1.148 -        for ( j = 0; j < MAX_NUMNODES; j++ )
   1.149 -            for ( k = 0; k <= MAX_ORDER; k++ )
   1.150 -                INIT_LIST_HEAD(&heap[i][j][k]);
   1.151 +    init_heap_block(&_heap0);
   1.152 +    _heap[0] = &_heap0;
   1.153 +    avail[0] = avail0;
   1.154  
   1.155      /* Pages that are free now go to the domain sub-allocator. */
   1.156      if ( (curr_free = next_free = !allocated_in_map(first_valid_mfn)) )
   1.157 @@ -1007,36 +1030,22 @@ unsigned long avail_scrub_pages(void)
   1.158      return scrub_pages;
   1.159  }
   1.160  
   1.161 -static unsigned long count_bucket(struct list_head* l, int order)
   1.162 -{
   1.163 -    unsigned long total_pages = 0;
   1.164 -    int pages = 1 << order;
   1.165 -    struct page_info *pg;
   1.166 -
   1.167 -    list_for_each_entry(pg, l, list)
   1.168 -        total_pages += pages;
   1.169 -
   1.170 -    return total_pages;
   1.171 -}
   1.172 -
   1.173  static void dump_heap(unsigned char key)
   1.174  {
   1.175 -    s_time_t       now = NOW();
   1.176 -    int i,j,k;
   1.177 -    unsigned long total;
   1.178 +    s_time_t      now = NOW();
   1.179 +    int           i, j;
   1.180  
   1.181      printk("'%c' pressed -> dumping heap info (now-0x%X:%08X)\n", key,
   1.182             (u32)(now>>32), (u32)now);
   1.183  
   1.184 -    for (i=0; i<NR_ZONES; i++ )
   1.185 -        for (j=0;j<MAX_NUMNODES;j++)
   1.186 -            for (k=0;k<=MAX_ORDER;k++)
   1.187 -                if ( !list_empty(&heap[i][j][k]) )
   1.188 -                {
   1.189 -                    total = count_bucket(&heap[i][j][k], k);
   1.190 -                    printk("heap[%d][%d][%d]-> %lu pages\n",
   1.191 -                            i, j, k, total);
   1.192 -                }
   1.193 +    for ( i = 0; i < MAX_NUMNODES; i++ )
   1.194 +    {
   1.195 +        if ( !avail[i] )
   1.196 +            continue;
   1.197 +        for ( j = 0; j < NR_ZONES; j++ )
   1.198 +            printk("heap[node=%d][zone=%d] -> %lu pages\n",
   1.199 +                   i, j, avail[i][j]);
   1.200 +    }
   1.201  }
   1.202  
   1.203  static __init int register_heap_trigger(void)