ia64/linux-2.6.18-xen.hg

annotate mm/bootmem.c @ 749:2892ca2b9c17

linux/x86: cleanup IO-APIC code

- get 32-bit code in sync with 64-bit wrt ExtINT pin detection being
unnecessary
- eliminate build warnings resulting from c/s 725

Signed-off-by: Jan Beulich <jbeulich@novell.com>
author Keir Fraser <keir.fraser@citrix.com>
date Fri Nov 28 13:31:21 2008 +0000 (2008-11-28)
parents 831230e53067
children
rev   line source
ian@0 1 /*
ian@0 2 * linux/mm/bootmem.c
ian@0 3 *
ian@0 4 * Copyright (C) 1999 Ingo Molnar
ian@0 5 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
ian@0 6 *
ian@0 7 * simple boot-time physical memory area allocator and
ian@0 8 * free memory collector. It's used to deal with reserved
ian@0 9 * system memory and memory holes as well.
ian@0 10 */
ian@0 11
ian@0 12 #include <linux/mm.h>
ian@0 13 #include <linux/kernel_stat.h>
ian@0 14 #include <linux/swap.h>
ian@0 15 #include <linux/interrupt.h>
ian@0 16 #include <linux/init.h>
ian@0 17 #include <linux/bootmem.h>
ian@0 18 #include <linux/mmzone.h>
ian@0 19 #include <linux/module.h>
ian@0 20 #include <asm/dma.h>
ian@0 21 #include <asm/io.h>
ian@0 22 #include "internal.h"
ian@0 23
ian@0 24 /*
ian@0 25 * Access to this subsystem has to be serialized externally. (this is
ian@0 26 * true for the boot process anyway)
ian@0 27 */
ian@0 28 unsigned long max_low_pfn;
ian@0 29 unsigned long min_low_pfn;
ian@0 30 unsigned long max_pfn;
ian@0 31
ian@0 32 EXPORT_UNUSED_SYMBOL(max_pfn); /* June 2006 */
ian@0 33
ian@0 34 static LIST_HEAD(bdata_list);
ian@0 35 #ifdef CONFIG_CRASH_DUMP
ian@0 36 /*
ian@0 37 * If we have booted due to a crash, max_pfn will be a very low value. We need
ian@0 38 * to know the amount of memory that the previous kernel used.
ian@0 39 */
ian@0 40 unsigned long saved_max_pfn;
ian@0 41 #endif
ian@0 42
ian@0 43 /* return the number of _pages_ that will be allocated for the boot bitmap */
ian@0 44 unsigned long __init bootmem_bootmap_pages (unsigned long pages)
ian@0 45 {
ian@0 46 unsigned long mapsize;
ian@0 47
ian@0 48 mapsize = (pages+7)/8;
ian@0 49 mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
ian@0 50 mapsize >>= PAGE_SHIFT;
ian@0 51
ian@0 52 return mapsize;
ian@0 53 }
ian@0 54 /*
ian@0 55 * link bdata in order
ian@0 56 */
ian@0 57 static void link_bootmem(bootmem_data_t *bdata)
ian@0 58 {
ian@0 59 bootmem_data_t *ent;
ian@0 60 if (list_empty(&bdata_list)) {
ian@0 61 list_add(&bdata->list, &bdata_list);
ian@0 62 return;
ian@0 63 }
ian@0 64 /* insert in order */
ian@0 65 list_for_each_entry(ent, &bdata_list, list) {
ian@0 66 if (bdata->node_boot_start < ent->node_boot_start) {
ian@0 67 list_add_tail(&bdata->list, &ent->list);
ian@0 68 return;
ian@0 69 }
ian@0 70 }
ian@0 71 list_add_tail(&bdata->list, &bdata_list);
ian@0 72 return;
ian@0 73 }
ian@0 74
ian@0 75
ian@0 76 /*
ian@0 77 * Called once to set up the allocator itself.
ian@0 78 */
ian@0 79 static unsigned long __init init_bootmem_core (pg_data_t *pgdat,
ian@0 80 unsigned long mapstart, unsigned long start, unsigned long end)
ian@0 81 {
ian@0 82 bootmem_data_t *bdata = pgdat->bdata;
ian@0 83 unsigned long mapsize = ((end - start)+7)/8;
ian@0 84
ian@0 85 mapsize = ALIGN(mapsize, sizeof(long));
ian@0 86 bdata->node_bootmem_map = phys_to_virt(mapstart << PAGE_SHIFT);
ian@0 87 bdata->node_boot_start = (start << PAGE_SHIFT);
ian@0 88 bdata->node_low_pfn = end;
ian@0 89 link_bootmem(bdata);
ian@0 90
ian@0 91 /*
ian@0 92 * Initially all pages are reserved - setup_arch() has to
ian@0 93 * register free RAM areas explicitly.
ian@0 94 */
ian@0 95 memset(bdata->node_bootmem_map, 0xff, mapsize);
ian@0 96
ian@0 97 return mapsize;
ian@0 98 }
ian@0 99
ian@0 100 /*
ian@0 101 * Marks a particular physical memory range as unallocatable. Usable RAM
ian@0 102 * might be used for boot-time allocations - or it might get added
ian@0 103 * to the free page pool later on.
ian@0 104 */
ian@0 105 static void __init reserve_bootmem_core(bootmem_data_t *bdata, unsigned long addr, unsigned long size)
ian@0 106 {
ian@0 107 unsigned long i;
ian@0 108 /*
ian@0 109 * round up, partially reserved pages are considered
ian@0 110 * fully reserved.
ian@0 111 */
ian@0 112 unsigned long sidx = (addr - bdata->node_boot_start)/PAGE_SIZE;
ian@0 113 unsigned long eidx = (addr + size - bdata->node_boot_start +
ian@0 114 PAGE_SIZE-1)/PAGE_SIZE;
ian@0 115 unsigned long end = (addr + size + PAGE_SIZE-1)/PAGE_SIZE;
ian@0 116
ian@0 117 BUG_ON(!size);
ian@0 118 BUG_ON(sidx >= eidx);
ian@0 119 BUG_ON((addr >> PAGE_SHIFT) >= bdata->node_low_pfn);
ian@0 120 BUG_ON(end > bdata->node_low_pfn);
ian@0 121
ian@0 122 for (i = sidx; i < eidx; i++)
ian@0 123 if (test_and_set_bit(i, bdata->node_bootmem_map)) {
ian@0 124 #ifdef CONFIG_DEBUG_BOOTMEM
ian@0 125 printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
ian@0 126 #endif
ian@0 127 }
ian@0 128 }
ian@0 129
ian@0 130 static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr, unsigned long size)
ian@0 131 {
ian@0 132 unsigned long i;
ian@0 133 unsigned long start;
ian@0 134 /*
ian@0 135 * round down end of usable mem, partially free pages are
ian@0 136 * considered reserved.
ian@0 137 */
ian@0 138 unsigned long sidx;
ian@0 139 unsigned long eidx = (addr + size - bdata->node_boot_start)/PAGE_SIZE;
ian@0 140 unsigned long end = (addr + size)/PAGE_SIZE;
ian@0 141
ian@0 142 BUG_ON(!size);
ian@0 143 BUG_ON(end > bdata->node_low_pfn);
ian@0 144
ian@0 145 if (addr < bdata->last_success)
ian@0 146 bdata->last_success = addr;
ian@0 147
ian@0 148 /*
ian@0 149 * Round up the beginning of the address.
ian@0 150 */
ian@0 151 start = (addr + PAGE_SIZE-1) / PAGE_SIZE;
ian@0 152 sidx = start - (bdata->node_boot_start/PAGE_SIZE);
ian@0 153
ian@0 154 for (i = sidx; i < eidx; i++) {
ian@0 155 if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
ian@0 156 BUG();
ian@0 157 }
ian@0 158 }
ian@0 159
ian@0 160 /*
ian@0 161 * We 'merge' subsequent allocations to save space. We might 'lose'
ian@0 162 * some fraction of a page if allocations cannot be satisfied due to
ian@0 163 * size constraints on boxes where there is physical RAM space
ian@0 164 * fragmentation - in these cases (mostly large memory boxes) this
ian@0 165 * is not a problem.
ian@0 166 *
ian@0 167 * On low memory boxes we get it right in 100% of the cases.
ian@0 168 *
ian@0 169 * alignment has to be a power of 2 value.
ian@0 170 *
ian@0 171 * NOTE: This function is _not_ reentrant.
ian@0 172 */
ian@0 173 void * __init
ian@0 174 __alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
ian@0 175 unsigned long align, unsigned long goal, unsigned long limit)
ian@0 176 {
ian@0 177 unsigned long offset, remaining_size, areasize, preferred;
ian@0 178 unsigned long i, start = 0, incr, eidx, end_pfn = bdata->node_low_pfn;
ian@0 179 void *ret;
ian@0 180
ian@0 181 if(!size) {
ian@0 182 printk("__alloc_bootmem_core(): zero-sized request\n");
ian@0 183 BUG();
ian@0 184 }
ian@0 185 BUG_ON(align & (align-1));
ian@0 186
ian@0 187 if (limit && bdata->node_boot_start >= limit)
ian@0 188 return NULL;
ian@0 189
ian@0 190 limit >>=PAGE_SHIFT;
ian@0 191 if (limit && end_pfn > limit)
ian@0 192 end_pfn = limit;
ian@0 193
ian@0 194 eidx = end_pfn - (bdata->node_boot_start >> PAGE_SHIFT);
ian@0 195 offset = 0;
ian@0 196 if (align &&
ian@0 197 (bdata->node_boot_start & (align - 1UL)) != 0)
ian@0 198 offset = (align - (bdata->node_boot_start & (align - 1UL)));
ian@0 199 offset >>= PAGE_SHIFT;
ian@0 200
ian@0 201 /*
ian@0 202 * We try to allocate bootmem pages above 'goal'
ian@0 203 * first, then we try to allocate lower pages.
ian@0 204 */
ian@0 205 if (goal && (goal >= bdata->node_boot_start) &&
ian@0 206 ((goal >> PAGE_SHIFT) < end_pfn)) {
ian@0 207 preferred = goal - bdata->node_boot_start;
ian@0 208
ian@0 209 if (bdata->last_success >= preferred)
ian@0 210 if (!limit || (limit && limit > bdata->last_success))
ian@0 211 preferred = bdata->last_success;
ian@0 212 } else
ian@0 213 preferred = 0;
ian@0 214
ian@0 215 preferred = ALIGN(preferred, align) >> PAGE_SHIFT;
ian@0 216 preferred += offset;
ian@0 217 areasize = (size+PAGE_SIZE-1)/PAGE_SIZE;
ian@0 218 incr = align >> PAGE_SHIFT ? : 1;
ian@0 219
ian@0 220 restart_scan:
ian@0 221 for (i = preferred; i < eidx; i += incr) {
ian@0 222 unsigned long j;
ian@0 223 i = find_next_zero_bit(bdata->node_bootmem_map, eidx, i);
ian@0 224 i = ALIGN(i, incr);
ian@0 225 if (i >= eidx)
ian@0 226 break;
ian@0 227 if (test_bit(i, bdata->node_bootmem_map))
ian@0 228 continue;
ian@0 229 for (j = i + 1; j < i + areasize; ++j) {
ian@0 230 if (j >= eidx)
ian@0 231 goto fail_block;
ian@0 232 if (test_bit (j, bdata->node_bootmem_map))
ian@0 233 goto fail_block;
ian@0 234 }
ian@0 235 start = i;
ian@0 236 goto found;
ian@0 237 fail_block:
ian@0 238 i = ALIGN(j, incr);
ian@0 239 }
ian@0 240
ian@0 241 if (preferred > offset) {
ian@0 242 preferred = offset;
ian@0 243 goto restart_scan;
ian@0 244 }
ian@0 245 return NULL;
ian@0 246
ian@0 247 found:
ian@0 248 bdata->last_success = start << PAGE_SHIFT;
ian@0 249 BUG_ON(start >= eidx);
ian@0 250
ian@0 251 /*
ian@0 252 * Is the next page of the previous allocation-end the start
ian@0 253 * of this allocation's buffer? If yes then we can 'merge'
ian@0 254 * the previous partial page with this allocation.
ian@0 255 */
ian@0 256 if (align < PAGE_SIZE &&
ian@0 257 bdata->last_offset && bdata->last_pos+1 == start) {
ian@0 258 offset = ALIGN(bdata->last_offset, align);
ian@0 259 BUG_ON(offset > PAGE_SIZE);
ian@0 260 remaining_size = PAGE_SIZE-offset;
ian@0 261 if (size < remaining_size) {
ian@0 262 areasize = 0;
ian@0 263 /* last_pos unchanged */
ian@0 264 bdata->last_offset = offset+size;
ian@0 265 ret = phys_to_virt(bdata->last_pos*PAGE_SIZE + offset +
ian@0 266 bdata->node_boot_start);
ian@0 267 } else {
ian@0 268 remaining_size = size - remaining_size;
ian@0 269 areasize = (remaining_size+PAGE_SIZE-1)/PAGE_SIZE;
ian@0 270 ret = phys_to_virt(bdata->last_pos*PAGE_SIZE + offset +
ian@0 271 bdata->node_boot_start);
ian@0 272 bdata->last_pos = start+areasize-1;
ian@0 273 bdata->last_offset = remaining_size;
ian@0 274 }
ian@0 275 bdata->last_offset &= ~PAGE_MASK;
ian@0 276 } else {
ian@0 277 bdata->last_pos = start + areasize - 1;
ian@0 278 bdata->last_offset = size & ~PAGE_MASK;
ian@0 279 ret = phys_to_virt(start * PAGE_SIZE + bdata->node_boot_start);
ian@0 280 }
ian@0 281
ian@0 282 /*
ian@0 283 * Reserve the area now:
ian@0 284 */
ian@0 285 for (i = start; i < start+areasize; i++)
ian@0 286 if (unlikely(test_and_set_bit(i, bdata->node_bootmem_map)))
ian@0 287 BUG();
ian@0 288 memset(ret, 0, size);
ian@0 289 return ret;
ian@0 290 }
ian@0 291
ian@0 292 static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
ian@0 293 {
ian@0 294 struct page *page;
ian@0 295 unsigned long pfn;
ian@0 296 bootmem_data_t *bdata = pgdat->bdata;
ian@0 297 unsigned long i, count, total = 0;
ian@0 298 unsigned long idx;
ian@0 299 unsigned long *map;
ian@0 300 int gofast = 0;
ian@0 301
ian@0 302 BUG_ON(!bdata->node_bootmem_map);
ian@0 303
ian@0 304 count = 0;
ian@0 305 /* first extant page of the node */
ian@0 306 pfn = bdata->node_boot_start >> PAGE_SHIFT;
ian@0 307 idx = bdata->node_low_pfn - (bdata->node_boot_start >> PAGE_SHIFT);
ian@0 308 map = bdata->node_bootmem_map;
ian@0 309 /* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
ian@0 310 if (bdata->node_boot_start == 0 ||
ian@0 311 ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG))
ian@0 312 gofast = 1;
ian@0 313 for (i = 0; i < idx; ) {
ian@0 314 unsigned long v = ~map[i / BITS_PER_LONG];
ian@0 315
ian@0 316 if (gofast && v == ~0UL) {
ian@0 317 int order;
ian@0 318
ian@0 319 page = pfn_to_page(pfn);
ian@0 320 count += BITS_PER_LONG;
ian@0 321 order = ffs(BITS_PER_LONG) - 1;
ian@0 322 __free_pages_bootmem(page, order);
ian@0 323 i += BITS_PER_LONG;
ian@0 324 page += BITS_PER_LONG;
ian@0 325 } else if (v) {
ian@0 326 unsigned long m;
ian@0 327
ian@0 328 page = pfn_to_page(pfn);
ian@0 329 for (m = 1; m && i < idx; m<<=1, page++, i++) {
ian@0 330 if (v & m) {
ian@0 331 count++;
ian@0 332 __free_pages_bootmem(page, 0);
ian@0 333 }
ian@0 334 }
ian@0 335 } else {
ian@0 336 i+=BITS_PER_LONG;
ian@0 337 }
ian@0 338 pfn += BITS_PER_LONG;
ian@0 339 }
ian@0 340 total += count;
ian@0 341
ian@0 342 /*
ian@0 343 * Now free the allocator bitmap itself, it's not
ian@0 344 * needed anymore:
ian@0 345 */
ian@0 346 page = virt_to_page(bdata->node_bootmem_map);
ian@0 347 count = 0;
ian@0 348 for (i = 0; i < ((bdata->node_low_pfn-(bdata->node_boot_start >> PAGE_SHIFT))/8 + PAGE_SIZE-1)/PAGE_SIZE; i++,page++) {
ian@0 349 count++;
ian@0 350 __free_pages_bootmem(page, 0);
ian@0 351 }
ian@0 352 total += count;
ian@0 353 bdata->node_bootmem_map = NULL;
ian@0 354
ian@0 355 return total;
ian@0 356 }
ian@0 357
ian@0 358 unsigned long __init init_bootmem_node (pg_data_t *pgdat, unsigned long freepfn, unsigned long startpfn, unsigned long endpfn)
ian@0 359 {
ian@0 360 return(init_bootmem_core(pgdat, freepfn, startpfn, endpfn));
ian@0 361 }
ian@0 362
ian@0 363 void __init reserve_bootmem_node (pg_data_t *pgdat, unsigned long physaddr, unsigned long size)
ian@0 364 {
ian@0 365 reserve_bootmem_core(pgdat->bdata, physaddr, size);
ian@0 366 }
ian@0 367
ian@0 368 void __init free_bootmem_node (pg_data_t *pgdat, unsigned long physaddr, unsigned long size)
ian@0 369 {
ian@0 370 free_bootmem_core(pgdat->bdata, physaddr, size);
ian@0 371 }
ian@0 372
ian@0 373 unsigned long __init free_all_bootmem_node (pg_data_t *pgdat)
ian@0 374 {
ian@0 375 return(free_all_bootmem_core(pgdat));
ian@0 376 }
ian@0 377
ian@0 378 unsigned long __init init_bootmem (unsigned long start, unsigned long pages)
ian@0 379 {
ian@0 380 max_low_pfn = pages;
ian@0 381 min_low_pfn = start;
ian@0 382 return(init_bootmem_core(NODE_DATA(0), start, 0, pages));
ian@0 383 }
ian@0 384
ian@0 385 #ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
ian@0 386 void __init reserve_bootmem (unsigned long addr, unsigned long size)
ian@0 387 {
ian@0 388 reserve_bootmem_core(NODE_DATA(0)->bdata, addr, size);
ian@0 389 }
ian@0 390 #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
ian@0 391
ian@0 392 void __init free_bootmem (unsigned long addr, unsigned long size)
ian@0 393 {
ian@0 394 free_bootmem_core(NODE_DATA(0)->bdata, addr, size);
ian@0 395 }
ian@0 396
ian@0 397 unsigned long __init free_all_bootmem (void)
ian@0 398 {
ian@0 399 return(free_all_bootmem_core(NODE_DATA(0)));
ian@0 400 }
ian@0 401
ian@0 402 void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align, unsigned long goal)
ian@0 403 {
ian@0 404 bootmem_data_t *bdata;
ian@0 405 void *ptr;
ian@0 406
ian@0 407 list_for_each_entry(bdata, &bdata_list, list)
ian@0 408 if ((ptr = __alloc_bootmem_core(bdata, size, align, goal, 0)))
ian@0 409 return(ptr);
ian@0 410 return NULL;
ian@0 411 }
ian@0 412
ian@0 413 void * __init __alloc_bootmem(unsigned long size, unsigned long align, unsigned long goal)
ian@0 414 {
ian@0 415 void *mem = __alloc_bootmem_nopanic(size,align,goal);
ian@0 416 if (mem)
ian@0 417 return mem;
ian@0 418 /*
ian@0 419 * Whoops, we cannot satisfy the allocation request.
ian@0 420 */
ian@0 421 printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
ian@0 422 panic("Out of memory");
ian@0 423 return NULL;
ian@0 424 }
ian@0 425
ian@0 426
ian@0 427 void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size, unsigned long align,
ian@0 428 unsigned long goal)
ian@0 429 {
ian@0 430 void *ptr;
ian@0 431
ian@0 432 ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
ian@0 433 if (ptr)
ian@0 434 return (ptr);
ian@0 435
ian@0 436 return __alloc_bootmem(size, align, goal);
ian@0 437 }
ian@0 438
ian@0 439 #define LOW32LIMIT 0xffffffff
ian@0 440
ian@0 441 void * __init __alloc_bootmem_low(unsigned long size, unsigned long align, unsigned long goal)
ian@0 442 {
ian@0 443 bootmem_data_t *bdata;
ian@0 444 void *ptr;
ian@0 445
ian@0 446 list_for_each_entry(bdata, &bdata_list, list)
ian@0 447 if ((ptr = __alloc_bootmem_core(bdata, size,
ian@0 448 align, goal, LOW32LIMIT)))
ian@0 449 return(ptr);
ian@0 450
ian@0 451 /*
ian@0 452 * Whoops, we cannot satisfy the allocation request.
ian@0 453 */
ian@0 454 printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
ian@0 455 panic("Out of low memory");
ian@0 456 return NULL;
ian@0 457 }
ian@0 458
ian@0 459 void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
ian@0 460 unsigned long align, unsigned long goal)
ian@0 461 {
ian@0 462 return __alloc_bootmem_core(pgdat->bdata, size, align, goal, LOW32LIMIT);
ian@0 463 }