direct-io.hg

view extras/mini-os/mm.c @ 11356:af7c87d42bc6

[XEN][POWERPC] Fix PHDR issues with large .data.percpu sections

This patch tells the link to only create one PHDR and place all sections
in it, also removing an unrequired mapping for the .data.percpu section.

This avoids the "Not enough room for program headers (allocated 2, need 3)"

Booted on a JS20.

Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
Signed-off-by: Hollis Blanchard <hollisb@us.ibm.com>
author Jimi Xenidis <jimix@watson.ibm.com>
date Thu Aug 17 07:10:57 2006 -0400 (2006-08-17)
parents cc991d9953d2
children e343f1850be0
line source
1 /*
2 ****************************************************************************
3 * (C) 2003 - Rolf Neugebauer - Intel Research Cambridge
4 * (C) 2005 - Grzegorz Milos - Intel Research Cambridge
5 ****************************************************************************
6 *
7 * File: mm.c
8 * Author: Rolf Neugebauer (neugebar@dcs.gla.ac.uk)
9 * Changes: Grzegorz Milos
10 *
11 * Date: Aug 2003, chages Aug 2005
12 *
13 * Environment: Xen Minimal OS
14 * Description: memory management related functions
15 * contains buddy page allocator from Xen.
16 *
17 ****************************************************************************
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this software and associated documentation files (the "Software"), to
20 * deal in the Software without restriction, including without limitation the
21 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
22 * sell copies of the Software, and to permit persons to whom the Software is
23 * furnished to do so, subject to the following conditions:
24 *
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
34 * DEALINGS IN THE SOFTWARE.
35 */
37 #include <os.h>
38 #include <hypervisor.h>
39 #include <mm.h>
40 #include <types.h>
41 #include <lib.h>
42 #include <xmalloc.h>
44 #ifdef MM_DEBUG
45 #define DEBUG(_f, _a...) \
46 printk("MINI_OS(file=mm.c, line=%d) " _f "\n", __LINE__, ## _a)
47 #else
48 #define DEBUG(_f, _a...) ((void)0)
49 #endif
51 unsigned long *phys_to_machine_mapping;
52 extern char *stack;
53 extern void page_walk(unsigned long virt_addr);
55 /*********************
56 * ALLOCATION BITMAP
57 * One bit per page of memory. Bit set => page is allocated.
58 */
60 static unsigned long *alloc_bitmap;
61 #define PAGES_PER_MAPWORD (sizeof(unsigned long) * 8)
63 #define allocated_in_map(_pn) \
64 (alloc_bitmap[(_pn)/PAGES_PER_MAPWORD] & (1<<((_pn)&(PAGES_PER_MAPWORD-1))))
66 /*
67 * Hint regarding bitwise arithmetic in map_{alloc,free}:
68 * -(1<<n) sets all bits >= n.
69 * (1<<n)-1 sets all bits < n.
70 * Variable names in map_{alloc,free}:
71 * *_idx == Index into `alloc_bitmap' array.
72 * *_off == Bit offset within an element of the `alloc_bitmap' array.
73 */
75 static void map_alloc(unsigned long first_page, unsigned long nr_pages)
76 {
77 unsigned long start_off, end_off, curr_idx, end_idx;
79 curr_idx = first_page / PAGES_PER_MAPWORD;
80 start_off = first_page & (PAGES_PER_MAPWORD-1);
81 end_idx = (first_page + nr_pages) / PAGES_PER_MAPWORD;
82 end_off = (first_page + nr_pages) & (PAGES_PER_MAPWORD-1);
84 if ( curr_idx == end_idx )
85 {
86 alloc_bitmap[curr_idx] |= ((1<<end_off)-1) & -(1<<start_off);
87 }
88 else
89 {
90 alloc_bitmap[curr_idx] |= -(1<<start_off);
91 while ( ++curr_idx < end_idx ) alloc_bitmap[curr_idx] = ~0L;
92 alloc_bitmap[curr_idx] |= (1<<end_off)-1;
93 }
94 }
97 static void map_free(unsigned long first_page, unsigned long nr_pages)
98 {
99 unsigned long start_off, end_off, curr_idx, end_idx;
101 curr_idx = first_page / PAGES_PER_MAPWORD;
102 start_off = first_page & (PAGES_PER_MAPWORD-1);
103 end_idx = (first_page + nr_pages) / PAGES_PER_MAPWORD;
104 end_off = (first_page + nr_pages) & (PAGES_PER_MAPWORD-1);
106 if ( curr_idx == end_idx )
107 {
108 alloc_bitmap[curr_idx] &= -(1<<end_off) | ((1<<start_off)-1);
109 }
110 else
111 {
112 alloc_bitmap[curr_idx] &= (1<<start_off)-1;
113 while ( ++curr_idx != end_idx ) alloc_bitmap[curr_idx] = 0;
114 alloc_bitmap[curr_idx] &= -(1<<end_off);
115 }
116 }
120 /*************************
121 * BINARY BUDDY ALLOCATOR
122 */
124 typedef struct chunk_head_st chunk_head_t;
125 typedef struct chunk_tail_st chunk_tail_t;
127 struct chunk_head_st {
128 chunk_head_t *next;
129 chunk_head_t **pprev;
130 int level;
131 };
133 struct chunk_tail_st {
134 int level;
135 };
137 /* Linked lists of free chunks of different powers-of-two in size. */
138 #define FREELIST_SIZE ((sizeof(void*)<<3)-PAGE_SHIFT)
139 static chunk_head_t *free_head[FREELIST_SIZE];
140 static chunk_head_t free_tail[FREELIST_SIZE];
141 #define FREELIST_EMPTY(_l) ((_l)->next == NULL)
143 #define round_pgdown(_p) ((_p)&PAGE_MASK)
144 #define round_pgup(_p) (((_p)+(PAGE_SIZE-1))&PAGE_MASK)
146 #ifdef MM_DEBUG
147 /*
148 * Prints allocation[0/1] for @nr_pages, starting at @start
149 * address (virtual).
150 */
151 static void print_allocation(void *start, int nr_pages)
152 {
153 unsigned long pfn_start = virt_to_pfn(start);
154 int count;
155 for(count = 0; count < nr_pages; count++)
156 if(allocated_in_map(pfn_start + count)) printk("1");
157 else printk("0");
159 printk("\n");
160 }
162 /*
163 * Prints chunks (making them with letters) for @nr_pages starting
164 * at @start (virtual).
165 */
166 static void print_chunks(void *start, int nr_pages)
167 {
168 char chunks[1001], current='A';
169 int order, count;
170 chunk_head_t *head;
171 unsigned long pfn_start = virt_to_pfn(start);
173 memset(chunks, (int)'_', 1000);
174 if(nr_pages > 1000)
175 {
176 DEBUG("Can only pring 1000 pages. Increase buffer size.");
177 }
179 for(order=0; order < FREELIST_SIZE; order++)
180 {
181 head = free_head[order];
182 while(!FREELIST_EMPTY(head))
183 {
184 for(count = 0; count < 1<< head->level; count++)
185 {
186 if(count + virt_to_pfn(head) - pfn_start < 1000)
187 chunks[count + virt_to_pfn(head) - pfn_start] = current;
188 }
189 head = head->next;
190 current++;
191 }
192 }
193 chunks[nr_pages] = '\0';
194 printk("%s\n", chunks);
195 }
196 #endif
199 /*
200 * Initialise allocator, placing addresses [@min,@max] in free pool.
201 * @min and @max are PHYSICAL addresses.
202 */
203 static void init_page_allocator(unsigned long min, unsigned long max)
204 {
205 int i;
206 unsigned long range, bitmap_size;
207 chunk_head_t *ch;
208 chunk_tail_t *ct;
209 for ( i = 0; i < FREELIST_SIZE; i++ )
210 {
211 free_head[i] = &free_tail[i];
212 free_tail[i].pprev = &free_head[i];
213 free_tail[i].next = NULL;
214 }
216 min = round_pgup (min);
217 max = round_pgdown(max);
219 /* Allocate space for the allocation bitmap. */
220 bitmap_size = (max+1) >> (PAGE_SHIFT+3);
221 bitmap_size = round_pgup(bitmap_size);
222 alloc_bitmap = (unsigned long *)to_virt(min);
223 min += bitmap_size;
224 range = max - min;
226 /* All allocated by default. */
227 memset(alloc_bitmap, ~0, bitmap_size);
228 /* Free up the memory we've been given to play with. */
229 map_free(min>>PAGE_SHIFT, range>>PAGE_SHIFT);
231 /* The buddy lists are addressed in high memory. */
232 min += VIRT_START;
233 max += VIRT_START;
235 while ( range != 0 )
236 {
237 /*
238 * Next chunk is limited by alignment of min, but also
239 * must not be bigger than remaining range.
240 */
241 for ( i = PAGE_SHIFT; (1<<(i+1)) <= range; i++ )
242 if ( min & (1<<i) ) break;
245 ch = (chunk_head_t *)min;
246 min += (1<<i);
247 range -= (1<<i);
248 ct = (chunk_tail_t *)min-1;
249 i -= PAGE_SHIFT;
250 ch->level = i;
251 ch->next = free_head[i];
252 ch->pprev = &free_head[i];
253 ch->next->pprev = &ch->next;
254 free_head[i] = ch;
255 ct->level = i;
256 }
257 }
260 /* Allocate 2^@order contiguous pages. Returns a VIRTUAL address. */
261 unsigned long alloc_pages(int order)
262 {
263 int i;
264 chunk_head_t *alloc_ch, *spare_ch;
265 chunk_tail_t *spare_ct;
268 /* Find smallest order which can satisfy the request. */
269 for ( i = order; i < FREELIST_SIZE; i++ ) {
270 if ( !FREELIST_EMPTY(free_head[i]) )
271 break;
272 }
274 if ( i == FREELIST_SIZE ) goto no_memory;
276 /* Unlink a chunk. */
277 alloc_ch = free_head[i];
278 free_head[i] = alloc_ch->next;
279 alloc_ch->next->pprev = alloc_ch->pprev;
281 /* We may have to break the chunk a number of times. */
282 while ( i != order )
283 {
284 /* Split into two equal parts. */
285 i--;
286 spare_ch = (chunk_head_t *)((char *)alloc_ch + (1<<(i+PAGE_SHIFT)));
287 spare_ct = (chunk_tail_t *)((char *)spare_ch + (1<<(i+PAGE_SHIFT)))-1;
289 /* Create new header for spare chunk. */
290 spare_ch->level = i;
291 spare_ch->next = free_head[i];
292 spare_ch->pprev = &free_head[i];
293 spare_ct->level = i;
295 /* Link in the spare chunk. */
296 spare_ch->next->pprev = &spare_ch->next;
297 free_head[i] = spare_ch;
298 }
300 map_alloc(to_phys(alloc_ch)>>PAGE_SHIFT, 1<<order);
302 return((unsigned long)alloc_ch);
304 no_memory:
306 printk("Cannot handle page request order %d!\n", order);
308 return 0;
309 }
311 void free_pages(void *pointer, int order)
312 {
313 chunk_head_t *freed_ch, *to_merge_ch;
314 chunk_tail_t *freed_ct;
315 unsigned long mask;
317 /* First free the chunk */
318 map_free(virt_to_pfn(pointer), 1 << order);
320 /* Create free chunk */
321 freed_ch = (chunk_head_t *)pointer;
322 freed_ct = (chunk_tail_t *)((char *)pointer + (1<<(order + PAGE_SHIFT)))-1;
324 /* Now, possibly we can conseal chunks together */
325 while(order < FREELIST_SIZE)
326 {
327 mask = 1 << (order + PAGE_SHIFT);
328 if((unsigned long)freed_ch & mask)
329 {
330 to_merge_ch = (chunk_head_t *)((char *)freed_ch - mask);
331 if(allocated_in_map(virt_to_pfn(to_merge_ch)) ||
332 to_merge_ch->level != order)
333 break;
335 /* Merge with predecessor */
336 freed_ch = to_merge_ch;
337 }
338 else
339 {
340 to_merge_ch = (chunk_head_t *)((char *)freed_ch + mask);
341 if(allocated_in_map(virt_to_pfn(to_merge_ch)) ||
342 to_merge_ch->level != order)
343 break;
345 /* Merge with successor */
346 freed_ct = (chunk_tail_t *)((char *)to_merge_ch + mask) - 1;
347 }
349 /* We are commited to merging, unlink the chunk */
350 *(to_merge_ch->pprev) = to_merge_ch->next;
351 to_merge_ch->next->pprev = to_merge_ch->pprev;
353 order++;
354 }
356 /* Link the new chunk */
357 freed_ch->level = order;
358 freed_ch->next = free_head[order];
359 freed_ch->pprev = &free_head[order];
360 freed_ct->level = order;
362 freed_ch->next->pprev = &freed_ch->next;
363 free_head[order] = freed_ch;
365 }
368 void new_pt_frame(unsigned long *pt_pfn, unsigned long prev_l_mfn,
369 unsigned long offset, unsigned long level)
370 {
371 pgentry_t *tab = (pgentry_t *)start_info.pt_base;
372 unsigned long pt_page = (unsigned long)pfn_to_virt(*pt_pfn);
373 unsigned long prot_e, prot_t, pincmd;
374 mmu_update_t mmu_updates[1];
375 struct mmuext_op pin_request;
377 DEBUG("Allocating new L%d pt frame for pt_pfn=%lx, "
378 "prev_l_mfn=%lx, offset=%lx",
379 level, *pt_pfn, prev_l_mfn, offset);
381 /* We need to clear the page, otherwise we might fail to map it
382 as a page table page */
383 memset((unsigned long*)pfn_to_virt(*pt_pfn), 0, PAGE_SIZE);
385 switch ( level )
386 {
387 case L1_FRAME:
388 prot_e = L1_PROT;
389 prot_t = L2_PROT;
390 pincmd = MMUEXT_PIN_L1_TABLE;
391 break;
392 #if defined(__x86_64__) || defined(CONFIG_X86_PAE)
393 case L2_FRAME:
394 prot_e = L2_PROT;
395 prot_t = L3_PROT;
396 pincmd = MMUEXT_PIN_L2_TABLE;
397 break;
398 #endif
399 #if defined(__x86_64__)
400 case L3_FRAME:
401 prot_e = L3_PROT;
402 prot_t = L4_PROT;
403 pincmd = MMUEXT_PIN_L3_TABLE;
404 break;
405 #endif
406 default:
407 printk("new_pt_frame() called with invalid level number %d\n", level);
408 do_exit();
409 break;
410 }
412 /* Update the entry */
413 #if defined(__x86_64__)
414 tab = pte_to_virt(tab[l4_table_offset(pt_page)]);
415 tab = pte_to_virt(tab[l3_table_offset(pt_page)]);
416 #endif
417 #if defined(CONFIG_X86_PAE)
418 tab = pte_to_virt(tab[l3_table_offset(pt_page)]);
419 #endif
421 mmu_updates[0].ptr = ((pgentry_t)tab[l2_table_offset(pt_page)] & PAGE_MASK) +
422 sizeof(pgentry_t) * l1_table_offset(pt_page);
423 mmu_updates[0].val = (pgentry_t)pfn_to_mfn(*pt_pfn) << PAGE_SHIFT |
424 (prot_e & ~_PAGE_RW);
425 if(HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF) < 0)
426 {
427 printk("PTE for new page table page could not be updated\n");
428 do_exit();
429 }
431 /* Pin the page to provide correct protection */
432 pin_request.cmd = pincmd;
433 pin_request.arg1.mfn = pfn_to_mfn(*pt_pfn);
434 if(HYPERVISOR_mmuext_op(&pin_request, 1, NULL, DOMID_SELF) < 0)
435 {
436 printk("ERROR: pinning failed\n");
437 do_exit();
438 }
440 /* Now fill the new page table page with entries.
441 Update the page directory as well. */
442 mmu_updates[0].ptr = ((pgentry_t)prev_l_mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
443 mmu_updates[0].val = (pgentry_t)pfn_to_mfn(*pt_pfn) << PAGE_SHIFT | prot_t;
444 if(HYPERVISOR_mmu_update(mmu_updates, 1, NULL, DOMID_SELF) < 0)
445 {
446 printk("ERROR: mmu_update failed\n");
447 do_exit();
448 }
450 *pt_pfn += 1;
451 }
453 /* Checks if a pagetable frame is needed (if weren't allocated by Xen) */
454 static int need_pt_frame(unsigned long virt_address, int level)
455 {
456 unsigned long hyp_virt_start = HYPERVISOR_VIRT_START;
457 #if defined(__x86_64__)
458 unsigned long hyp_virt_end = HYPERVISOR_VIRT_END;
459 #else
460 unsigned long hyp_virt_end = 0xffffffff;
461 #endif
463 /* In general frames will _not_ be needed if they were already
464 allocated to map the hypervisor into our VA space */
465 #if defined(__x86_64__)
466 if(level == L3_FRAME)
467 {
468 if(l4_table_offset(virt_address) >=
469 l4_table_offset(hyp_virt_start) &&
470 l4_table_offset(virt_address) <=
471 l4_table_offset(hyp_virt_end))
472 return 0;
473 return 1;
474 } else
475 #endif
477 #if defined(__x86_64__) || defined(CONFIG_X86_PAE)
478 if(level == L2_FRAME)
479 {
480 #if defined(__x86_64__)
481 if(l4_table_offset(virt_address) >=
482 l4_table_offset(hyp_virt_start) &&
483 l4_table_offset(virt_address) <=
484 l4_table_offset(hyp_virt_end))
485 #endif
486 if(l3_table_offset(virt_address) >=
487 l3_table_offset(hyp_virt_start) &&
488 l3_table_offset(virt_address) <=
489 l3_table_offset(hyp_virt_end))
490 return 0;
492 return 1;
493 } else
494 #endif /* defined(__x86_64__) || defined(CONFIG_X86_PAE) */
496 /* Always need l1 frames */
497 if(level == L1_FRAME)
498 return 1;
500 printk("ERROR: Unknown frame level %d, hypervisor %llx,%llx\n",
501 level, hyp_virt_start, hyp_virt_end);
502 return -1;
503 }
505 void build_pagetable(unsigned long *start_pfn, unsigned long *max_pfn)
506 {
507 unsigned long start_address, end_address;
508 unsigned long pfn_to_map, pt_pfn = *start_pfn;
509 static mmu_update_t mmu_updates[L1_PAGETABLE_ENTRIES + 1];
510 pgentry_t *tab = (pgentry_t *)start_info.pt_base, page;
511 unsigned long mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
512 unsigned long offset;
513 int count = 0;
515 pfn_to_map = (start_info.nr_pt_frames - NOT_L1_FRAMES) * L1_PAGETABLE_ENTRIES;
517 if (*max_pfn >= virt_to_pfn(HYPERVISOR_VIRT_START))
518 {
519 printk("WARNING: Mini-OS trying to use Xen virtual space. "
520 "Truncating memory from %dMB to ",
521 ((unsigned long)pfn_to_virt(*max_pfn) - (unsigned long)&_text)>>20);
522 *max_pfn = virt_to_pfn(HYPERVISOR_VIRT_START - PAGE_SIZE);
523 printk("%dMB\n",
524 ((unsigned long)pfn_to_virt(*max_pfn) - (unsigned long)&_text)>>20);
525 }
527 start_address = (unsigned long)pfn_to_virt(pfn_to_map);
528 end_address = (unsigned long)pfn_to_virt(*max_pfn);
530 /* We worked out the virtual memory range to map, now mapping loop */
531 printk("Mapping memory range 0x%lx - 0x%lx\n", start_address, end_address);
533 while(start_address < end_address)
534 {
535 tab = (pgentry_t *)start_info.pt_base;
536 mfn = pfn_to_mfn(virt_to_pfn(start_info.pt_base));
538 #if defined(__x86_64__)
539 offset = l4_table_offset(start_address);
540 /* Need new L3 pt frame */
541 if(!(start_address & L3_MASK))
542 if(need_pt_frame(start_address, L3_FRAME))
543 new_pt_frame(&pt_pfn, mfn, offset, L3_FRAME);
545 page = tab[offset];
546 mfn = pte_to_mfn(page);
547 tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT);
548 #endif
549 #if defined(__x86_64__) || defined(CONFIG_X86_PAE)
550 offset = l3_table_offset(start_address);
551 /* Need new L2 pt frame */
552 if(!(start_address & L2_MASK))
553 if(need_pt_frame(start_address, L2_FRAME))
554 new_pt_frame(&pt_pfn, mfn, offset, L2_FRAME);
556 page = tab[offset];
557 mfn = pte_to_mfn(page);
558 tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT);
559 #endif
560 offset = l2_table_offset(start_address);
561 /* Need new L1 pt frame */
562 if(!(start_address & L1_MASK))
563 if(need_pt_frame(start_address, L1_FRAME))
564 new_pt_frame(&pt_pfn, mfn, offset, L1_FRAME);
566 page = tab[offset];
567 mfn = pte_to_mfn(page);
568 offset = l1_table_offset(start_address);
570 mmu_updates[count].ptr = ((pgentry_t)mfn << PAGE_SHIFT) + sizeof(pgentry_t) * offset;
571 mmu_updates[count].val = (pgentry_t)pfn_to_mfn(pfn_to_map++) << PAGE_SHIFT | L1_PROT;
572 count++;
573 if (count == L1_PAGETABLE_ENTRIES || pfn_to_map == *max_pfn)
574 {
575 if(HYPERVISOR_mmu_update(mmu_updates, count, NULL, DOMID_SELF) < 0)
576 {
577 printk("PTE could not be updated\n");
578 do_exit();
579 }
580 count = 0;
581 }
582 start_address += PAGE_SIZE;
583 }
585 *start_pfn = pt_pfn;
586 }
589 void mem_test(unsigned long *start_add, unsigned long *end_add)
590 {
591 unsigned long mask = 0x10000;
592 unsigned long *pointer;
594 for(pointer = start_add; pointer < end_add; pointer++)
595 {
596 if(!(((unsigned long)pointer) & 0xfffff))
597 {
598 printk("Writing to %lx\n", pointer);
599 page_walk((unsigned long)pointer);
600 }
601 *pointer = (unsigned long)pointer & ~mask;
602 }
604 for(pointer = start_add; pointer < end_add; pointer++)
605 {
606 if(((unsigned long)pointer & ~mask) != *pointer)
607 printk("Read error at 0x%lx. Read: 0x%lx, should read 0x%lx\n",
608 (unsigned long)pointer,
609 *pointer,
610 ((unsigned long)pointer & ~mask));
611 }
613 }
615 static pgentry_t *demand_map_pgt;
616 static void *demand_map_area_start;
618 static void init_demand_mapping_area(unsigned long max_pfn)
619 {
620 unsigned long mfn;
621 pgentry_t *tab;
622 unsigned long start_addr;
623 unsigned long pt_pfn;
624 unsigned offset;
626 /* Round up to four megs. + 1024 rather than + 1023 since we want
627 to be sure we don't end up in the same place we started. */
628 max_pfn = (max_pfn + L1_PAGETABLE_ENTRIES) & ~(L1_PAGETABLE_ENTRIES - 1);
629 if (max_pfn == 0 ||
630 (unsigned long)pfn_to_virt(max_pfn + L1_PAGETABLE_ENTRIES) >=
631 HYPERVISOR_VIRT_START) {
632 printk("Too much memory; no room for demand map hole.\n");
633 do_exit();
634 }
636 demand_map_area_start = pfn_to_virt(max_pfn);
637 printk("Demand map pfns start at %lx (%p).\n", max_pfn,
638 demand_map_area_start);
639 start_addr = (unsigned long)demand_map_area_start;
641 tab = (pgentry_t *)start_info.pt_base;
642 mfn = virt_to_mfn(start_info.pt_base);
643 pt_pfn = virt_to_pfn(alloc_page());
645 #if defined(__x86_64__)
646 offset = l4_table_offset(start_addr);
647 if (!(tab[offset] & _PAGE_PRESENT)) {
648 new_pt_frame(&pt_pfn, mfn, offset, L3_FRAME);
649 pt_pfn = virt_to_pfn(alloc_page());
650 }
651 ASSERT(tab[offset] & _PAGE_PRESENT);
652 mfn = pte_to_mfn(tab[offset]);
653 tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT);
654 #endif
655 #if defined(__x86_64__) || defined(CONFIG_X86_PAE)
656 offset = l3_table_offset(start_addr);
657 if (!(tab[offset] & _PAGE_PRESENT)) {
658 new_pt_frame(&pt_pfn, mfn, offset, L2_FRAME);
659 pt_pfn = virt_to_pfn(alloc_page());
660 }
661 ASSERT(tab[offset] & _PAGE_PRESENT);
662 mfn = pte_to_mfn(tab[offset]);
663 tab = to_virt(mfn_to_pfn(mfn) << PAGE_SHIFT);
664 #endif
665 offset = l2_table_offset(start_addr);
666 if (tab[offset] & _PAGE_PRESENT) {
667 printk("Demand map area already has a page table covering it?\n");
668 BUG();
669 }
670 demand_map_pgt = pfn_to_virt(pt_pfn);
671 new_pt_frame(&pt_pfn, mfn, offset, L1_FRAME);
672 ASSERT(tab[offset] & _PAGE_PRESENT);
673 }
675 void *map_frames(unsigned long *f, unsigned long n)
676 {
677 unsigned long x;
678 unsigned long y = 0;
679 mmu_update_t mmu_updates[16];
680 int rc;
682 if (n > 16) {
683 printk("Tried to map too many (%ld) frames at once.\n", n);
684 return NULL;
685 }
687 /* Find a run of n contiguous frames */
688 for (x = 0; x <= 1024 - n; x += y + 1) {
689 for (y = 0; y < n; y++)
690 if (demand_map_pgt[x+y] & _PAGE_PRESENT)
691 break;
692 if (y == n)
693 break;
694 }
695 if (y != n) {
696 printk("Failed to map %ld frames!\n", n);
697 return NULL;
698 }
700 /* Found it at x. Map it in. */
701 for (y = 0; y < n; y++) {
702 mmu_updates[y].ptr = virt_to_mach(&demand_map_pgt[x + y]);
703 mmu_updates[y].val = (f[y] << PAGE_SHIFT) | L1_PROT;
704 }
706 rc = HYPERVISOR_mmu_update(mmu_updates, n, NULL, DOMID_SELF);
707 if (rc < 0) {
708 printk("Map %ld failed: %d.\n", n, rc);
709 return NULL;
710 } else {
711 return (void *)(unsigned long)((unsigned long)demand_map_area_start +
712 x * PAGE_SIZE);
713 }
714 }
716 void init_mm(void)
717 {
719 unsigned long start_pfn, max_pfn;
721 printk("MM: Init\n");
723 printk(" _text: %p\n", &_text);
724 printk(" _etext: %p\n", &_etext);
725 printk(" _edata: %p\n", &_edata);
726 printk(" stack start: %p\n", &stack);
727 printk(" _end: %p\n", &_end);
729 /* First page follows page table pages and 3 more pages (store page etc) */
730 start_pfn = PFN_UP(to_phys(start_info.pt_base)) +
731 start_info.nr_pt_frames + 3;
732 max_pfn = start_info.nr_pages;
734 printk(" start_pfn: %lx\n", start_pfn);
735 printk(" max_pfn: %lx\n", max_pfn);
737 build_pagetable(&start_pfn, &max_pfn);
739 /*
740 * now we can initialise the page allocator
741 */
742 printk("MM: Initialise page allocator for %lx(%lx)-%lx(%lx)\n",
743 (u_long)to_virt(PFN_PHYS(start_pfn)), PFN_PHYS(start_pfn),
744 (u_long)to_virt(PFN_PHYS(max_pfn)), PFN_PHYS(max_pfn));
745 init_page_allocator(PFN_PHYS(start_pfn), PFN_PHYS(max_pfn));
746 printk("MM: done\n");
748 init_demand_mapping_area(max_pfn);
749 printk("Initialised demand area.\n");
750 }
752 void sanity_check(void)
753 {
754 int x;
755 chunk_head_t *head;
757 for (x = 0; x < FREELIST_SIZE; x++) {
758 for (head = free_head[x]; !FREELIST_EMPTY(head); head = head->next) {
759 ASSERT(!allocated_in_map(virt_to_pfn(head)));
760 if (head->next)
761 ASSERT(head->next->pprev == &head->next);
762 }
763 if (free_head[x]) {
764 ASSERT(free_head[x]->pprev == &free_head[x]);
765 }
766 }
767 }