ia64/xen-unstable

view xen/common/grant_table.c @ 6317:2d4daffd8a4a

Fix fallout from "Get rid of memory_t." checkin.
{get,put}_user of u64 items has to use u64 variables.
Signed-off-by: Christian Limpach <Christian.Limpach@cl.cam.ac.uk>
author cl349@firebug.cl.cam.ac.uk
date Sun Aug 21 11:40:39 2005 +0000 (2005-08-21)
parents 35f3f9424c93
children 6721abf6b16d
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #define GRANT_DEBUG 0
28 #define GRANT_DEBUG_VERBOSE 0
30 #include <xen/config.h>
31 #include <xen/lib.h>
32 #include <xen/sched.h>
33 #include <xen/shadow.h>
34 #include <xen/mm.h>
35 #include <acm/acm_hooks.h>
37 #if defined(CONFIG_X86_64)
38 #define GRANT_PTE_FLAGS (_PAGE_PRESENT|_PAGE_ACCESSED|_PAGE_DIRTY|_PAGE_USER)
39 #else
40 #define GRANT_PTE_FLAGS (_PAGE_PRESENT|_PAGE_ACCESSED|_PAGE_DIRTY)
41 #endif
43 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
44 do { \
45 DPRINTK( _f, ## _a ); \
46 rc = (_rc); \
47 goto _lbl; \
48 } while ( 0 )
50 static inline int
51 get_maptrack_handle(
52 grant_table_t *t)
53 {
54 unsigned int h;
55 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
56 return -1;
57 t->maptrack_head = t->maptrack[h].ref_and_flags >> MAPTRACK_REF_SHIFT;
58 t->map_count++;
59 return h;
60 }
62 static inline void
63 put_maptrack_handle(
64 grant_table_t *t, int handle)
65 {
66 t->maptrack[handle].ref_and_flags = t->maptrack_head << MAPTRACK_REF_SHIFT;
67 t->maptrack_head = handle;
68 t->map_count--;
69 }
71 static int
72 __gnttab_activate_grant_ref(
73 struct domain *mapping_d, /* IN */
74 struct vcpu *mapping_ed,
75 struct domain *granting_d,
76 grant_ref_t ref,
77 u16 dev_hst_ro_flags,
78 u64 addr,
79 u64 *pframe ) /* OUT */
80 {
81 domid_t sdom;
82 u16 sflags;
83 active_grant_entry_t *act;
84 grant_entry_t *sha;
85 s16 rc = 1;
86 unsigned long frame = 0;
87 int retries = 0;
89 /*
90 * Objectives of this function:
91 * . Make the record ( granting_d, ref ) active, if not already.
92 * . Update shared grant entry of owner, indicating frame is mapped.
93 * . Increment the owner act->pin reference counts.
94 * . get_page on shared frame if new mapping.
95 * . get_page_type if this is first RW mapping of frame.
96 * . Add PTE to virtual address space of mapping_d, if necessary.
97 * Returns:
98 * . -ve: error
99 * . 1: ok
100 * . 0: ok and TLB invalidate of host_addr needed.
101 *
102 * On success, *pframe contains mfn.
103 */
105 /*
106 * We bound the number of times we retry CMPXCHG on memory locations that
107 * we share with a guest OS. The reason is that the guest can modify that
108 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
109 * could cause us to livelock. There are a few cases where it is valid for
110 * the guest to race our updates (e.g., to change the GTF_readonly flag),
111 * so we allow a few retries before failing.
112 */
114 act = &granting_d->grant_table->active[ref];
115 sha = &granting_d->grant_table->shared[ref];
117 spin_lock(&granting_d->grant_table->lock);
119 if ( act->pin == 0 )
120 {
121 /* CASE 1: Activating a previously inactive entry. */
123 sflags = sha->flags;
124 sdom = sha->domid;
126 /* This loop attempts to set the access (reading/writing) flags
127 * in the grant table entry. It tries a cmpxchg on the field
128 * up to five times, and then fails under the assumption that
129 * the guest is misbehaving. */
130 for ( ; ; )
131 {
132 u32 scombo, prev_scombo, new_scombo;
134 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
135 unlikely(sdom != mapping_d->domain_id) )
136 PIN_FAIL(unlock_out, GNTST_general_error,
137 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
138 sflags, sdom, mapping_d->domain_id);
140 /* Merge two 16-bit values into a 32-bit combined update. */
141 /* NB. Endianness! */
142 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
144 new_scombo = scombo | GTF_reading;
145 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
146 {
147 new_scombo |= GTF_writing;
148 if ( unlikely(sflags & GTF_readonly) )
149 PIN_FAIL(unlock_out, GNTST_general_error,
150 "Attempt to write-pin a r/o grant entry.\n");
151 }
153 /* NB. prev_scombo is updated in place to seen value. */
154 if ( unlikely(cmpxchg_user((u32 *)&sha->flags,
155 prev_scombo,
156 new_scombo)) )
157 PIN_FAIL(unlock_out, GNTST_general_error,
158 "Fault while modifying shared flags and domid.\n");
160 /* Did the combined update work (did we see what we expected?). */
161 if ( likely(prev_scombo == scombo) )
162 break;
164 if ( retries++ == 4 )
165 PIN_FAIL(unlock_out, GNTST_general_error,
166 "Shared grant entry is unstable.\n");
168 /* Didn't see what we expected. Split out the seen flags & dom. */
169 /* NB. Endianness! */
170 sflags = (u16)prev_scombo;
171 sdom = (u16)(prev_scombo >> 16);
172 }
174 /* rmb(); */ /* not on x86 */
176 frame = __gpfn_to_mfn_foreign(granting_d, sha->frame);
178 if ( unlikely(!pfn_valid(frame)) ||
179 unlikely(!((dev_hst_ro_flags & GNTMAP_readonly) ?
180 get_page(&frame_table[frame], granting_d) :
181 get_page_and_type(&frame_table[frame], granting_d,
182 PGT_writable_page))) )
183 {
184 clear_bit(_GTF_writing, &sha->flags);
185 clear_bit(_GTF_reading, &sha->flags);
186 PIN_FAIL(unlock_out, GNTST_general_error,
187 "Could not pin the granted frame (%lx)!\n", frame);
188 }
190 if ( dev_hst_ro_flags & GNTMAP_device_map )
191 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
192 GNTPIN_devr_inc : GNTPIN_devw_inc;
193 if ( dev_hst_ro_flags & GNTMAP_host_map )
194 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
195 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
196 act->domid = sdom;
197 act->frame = frame;
198 }
199 else
200 {
201 /* CASE 2: Active modications to an already active entry. */
203 /*
204 * A cheesy check for possible pin-count overflow.
205 * A more accurate check cannot be done with a single comparison.
206 */
207 if ( (act->pin & 0x80808080U) != 0 )
208 PIN_FAIL(unlock_out, ENOSPC,
209 "Risk of counter overflow %08x\n", act->pin);
211 frame = act->frame;
213 if ( !(dev_hst_ro_flags & GNTMAP_readonly) &&
214 !((sflags = sha->flags) & GTF_writing) )
215 {
216 for ( ; ; )
217 {
218 u16 prev_sflags;
220 if ( unlikely(sflags & GTF_readonly) )
221 PIN_FAIL(unlock_out, GNTST_general_error,
222 "Attempt to write-pin a r/o grant entry.\n");
224 prev_sflags = sflags;
226 /* NB. prev_sflags is updated in place to seen value. */
227 if ( unlikely(cmpxchg_user(&sha->flags, prev_sflags,
228 prev_sflags | GTF_writing)) )
229 PIN_FAIL(unlock_out, GNTST_general_error,
230 "Fault while modifying shared flags.\n");
232 if ( likely(prev_sflags == sflags) )
233 break;
235 if ( retries++ == 4 )
236 PIN_FAIL(unlock_out, GNTST_general_error,
237 "Shared grant entry is unstable.\n");
239 sflags = prev_sflags;
240 }
242 if ( unlikely(!get_page_type(&frame_table[frame],
243 PGT_writable_page)) )
244 {
245 clear_bit(_GTF_writing, &sha->flags);
246 PIN_FAIL(unlock_out, GNTST_general_error,
247 "Attempt to write-pin a unwritable page.\n");
248 }
249 }
251 if ( dev_hst_ro_flags & GNTMAP_device_map )
252 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
253 GNTPIN_devr_inc : GNTPIN_devw_inc;
255 if ( dev_hst_ro_flags & GNTMAP_host_map )
256 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
257 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
258 }
260 /*
261 * At this point:
262 * act->pin updated to reference count mappings.
263 * sha->flags updated to indicate to granting domain mapping done.
264 * frame contains the mfn.
265 */
267 spin_unlock(&granting_d->grant_table->lock);
269 if ( (addr != 0) && (dev_hst_ro_flags & GNTMAP_host_map) )
270 {
271 /* Write update into the pagetable. */
272 l1_pgentry_t pte;
273 pte = l1e_from_pfn(frame, GRANT_PTE_FLAGS);
275 if ( (dev_hst_ro_flags & GNTMAP_application_map) )
276 l1e_add_flags(pte,_PAGE_USER);
277 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
278 l1e_add_flags(pte,_PAGE_RW);
280 if ( dev_hst_ro_flags & GNTMAP_contains_pte )
281 rc = update_grant_pte_mapping(addr, pte, mapping_d, mapping_ed);
282 else
283 rc = update_grant_va_mapping(addr, pte, mapping_d, mapping_ed);
285 /* IMPORTANT: rc indicates the degree of TLB flush that is required.
286 * GNTST_flush_one (1) or GNTST_flush_all (2). This is done in the
287 * outer gnttab_map_grant_ref. */
288 if ( rc < 0 )
289 {
290 /* Failure: undo and abort. */
292 spin_lock(&granting_d->grant_table->lock);
294 if ( dev_hst_ro_flags & GNTMAP_readonly )
295 {
296 act->pin -= GNTPIN_hstr_inc;
297 }
298 else
299 {
300 act->pin -= GNTPIN_hstw_inc;
301 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
302 {
303 clear_bit(_GTF_writing, &sha->flags);
304 put_page_type(&frame_table[frame]);
305 }
306 }
308 if ( act->pin == 0 )
309 {
310 clear_bit(_GTF_reading, &sha->flags);
311 put_page(&frame_table[frame]);
312 }
314 spin_unlock(&granting_d->grant_table->lock);
315 }
317 }
319 *pframe = frame;
320 return rc;
322 unlock_out:
323 spin_unlock(&granting_d->grant_table->lock);
324 return rc;
325 }
327 /*
328 * Returns 0 if TLB flush / invalidate required by caller.
329 * va will indicate the address to be invalidated.
330 *
331 * addr is _either_ a host virtual address, or the address of the pte to
332 * update, as indicated by the GNTMAP_contains_pte flag.
333 */
334 static int
335 __gnttab_map_grant_ref(
336 gnttab_map_grant_ref_t *uop,
337 unsigned long *va)
338 {
339 domid_t dom;
340 grant_ref_t ref;
341 struct domain *ld, *rd;
342 struct vcpu *led;
343 u16 dev_hst_ro_flags;
344 int handle;
345 u64 frame = 0, addr;
346 int rc;
348 led = current;
349 ld = led->domain;
351 /* Bitwise-OR avoids short-circuiting which screws control flow. */
352 if ( unlikely(__get_user(dom, &uop->dom) |
353 __get_user(ref, &uop->ref) |
354 __get_user(addr, &uop->host_addr) |
355 __get_user(dev_hst_ro_flags, &uop->flags)) )
356 {
357 DPRINTK("Fault while reading gnttab_map_grant_ref_t.\n");
358 return -EFAULT; /* don't set status */
359 }
361 if ( (dev_hst_ro_flags & GNTMAP_host_map) &&
362 ( (addr == 0) ||
363 (!(dev_hst_ro_flags & GNTMAP_contains_pte) &&
364 unlikely(!__addr_ok(addr))) ) )
365 {
366 DPRINTK("Bad virtual address (%llx) or flags (%x).\n",
367 addr, dev_hst_ro_flags);
368 (void)__put_user(GNTST_bad_virt_addr, &uop->handle);
369 return GNTST_bad_gntref;
370 }
372 if ( unlikely(ref >= NR_GRANT_ENTRIES) ||
373 unlikely((dev_hst_ro_flags &
374 (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
375 {
376 DPRINTK("Bad ref (%d) or flags (%x).\n", ref, dev_hst_ro_flags);
377 (void)__put_user(GNTST_bad_gntref, &uop->handle);
378 return GNTST_bad_gntref;
379 }
381 if (acm_pre_grant_map_ref(dom)) {
382 (void)__put_user(GNTST_permission_denied, &uop->handle);
383 return GNTST_permission_denied;
384 }
386 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
387 unlikely(ld == rd) )
388 {
389 if ( rd != NULL )
390 put_domain(rd);
391 DPRINTK("Could not find domain %d\n", dom);
392 (void)__put_user(GNTST_bad_domain, &uop->handle);
393 return GNTST_bad_domain;
394 }
396 /* Get a maptrack handle. */
397 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
398 {
399 int i;
400 grant_mapping_t *new_mt;
401 grant_table_t *lgt = ld->grant_table;
403 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
404 {
405 put_domain(rd);
406 DPRINTK("Maptrack table is at maximum size.\n");
407 (void)__put_user(GNTST_no_device_space, &uop->handle);
408 return GNTST_no_device_space;
409 }
411 /* Grow the maptrack table. */
412 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
413 if ( new_mt == NULL )
414 {
415 put_domain(rd);
416 DPRINTK("No more map handles available.\n");
417 (void)__put_user(GNTST_no_device_space, &uop->handle);
418 return GNTST_no_device_space;
419 }
421 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
422 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
423 new_mt[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
425 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
426 lgt->maptrack = new_mt;
427 lgt->maptrack_order += 1;
428 lgt->maptrack_limit <<= 1;
430 DPRINTK("Doubled maptrack size\n");
431 handle = get_maptrack_handle(ld->grant_table);
432 }
434 #if GRANT_DEBUG_VERBOSE
435 DPRINTK("Mapping grant ref (%hu) for domain (%hu) with flags (%x)\n",
436 ref, dom, dev_hst_ro_flags);
437 #endif
439 if ( 0 <= ( rc = __gnttab_activate_grant_ref( ld, led, rd, ref,
440 dev_hst_ro_flags,
441 addr, &frame)))
442 {
443 /*
444 * Only make the maptrack live _after_ writing the pte, in case we
445 * overwrite the same frame number, causing a maptrack walk to find it
446 */
447 ld->grant_table->maptrack[handle].domid = dom;
449 ld->grant_table->maptrack[handle].ref_and_flags
450 = (ref << MAPTRACK_REF_SHIFT) |
451 (dev_hst_ro_flags & MAPTRACK_GNTMAP_MASK);
453 (void)__put_user(frame, &uop->dev_bus_addr);
455 if ( ( dev_hst_ro_flags & GNTMAP_host_map ) &&
456 !( dev_hst_ro_flags & GNTMAP_contains_pte) )
457 *va = addr;
459 (void)__put_user(handle, &uop->handle);
460 }
461 else
462 {
463 (void)__put_user(rc, &uop->handle);
464 put_maptrack_handle(ld->grant_table, handle);
465 }
467 put_domain(rd);
468 return rc;
469 }
471 static long
472 gnttab_map_grant_ref(
473 gnttab_map_grant_ref_t *uop, unsigned int count)
474 {
475 int i, rc, flush = 0;
476 unsigned long va = 0;
478 for ( i = 0; i < count; i++ )
479 if ( (rc =__gnttab_map_grant_ref(&uop[i], &va)) >= 0 )
480 flush += rc;
482 if ( flush == 1 )
483 flush_tlb_one_mask(current->domain->cpumask, va);
484 else if ( flush != 0 )
485 flush_tlb_mask(current->domain->cpumask);
487 return 0;
488 }
490 static int
491 __gnttab_unmap_grant_ref(
492 gnttab_unmap_grant_ref_t *uop,
493 unsigned long *va)
494 {
495 domid_t dom;
496 grant_ref_t ref;
497 u16 handle;
498 struct domain *ld, *rd;
500 active_grant_entry_t *act;
501 grant_entry_t *sha;
502 grant_mapping_t *map;
503 u16 flags;
504 s16 rc = 1;
505 u64 frame, addr;
507 ld = current->domain;
509 /* Bitwise-OR avoids short-circuiting which screws control flow. */
510 if ( unlikely(__get_user(addr, &uop->host_addr) |
511 __get_user(frame, &uop->dev_bus_addr) |
512 __get_user(handle, &uop->handle)) )
513 {
514 DPRINTK("Fault while reading gnttab_unmap_grant_ref_t.\n");
515 return -EFAULT; /* don't set status */
516 }
518 map = &ld->grant_table->maptrack[handle];
520 if ( unlikely(handle >= ld->grant_table->maptrack_limit) ||
521 unlikely(!(map->ref_and_flags & MAPTRACK_GNTMAP_MASK)) )
522 {
523 DPRINTK("Bad handle (%d).\n", handle);
524 (void)__put_user(GNTST_bad_handle, &uop->status);
525 return GNTST_bad_handle;
526 }
528 dom = map->domid;
529 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
530 flags = map->ref_and_flags & MAPTRACK_GNTMAP_MASK;
532 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
533 unlikely(ld == rd) )
534 {
535 if ( rd != NULL )
536 put_domain(rd);
537 DPRINTK("Could not find domain %d\n", dom);
538 (void)__put_user(GNTST_bad_domain, &uop->status);
539 return GNTST_bad_domain;
540 }
542 #if GRANT_DEBUG_VERBOSE
543 DPRINTK("Unmapping grant ref (%hu) for domain (%hu) with handle (%hu)\n",
544 ref, dom, handle);
545 #endif
547 act = &rd->grant_table->active[ref];
548 sha = &rd->grant_table->shared[ref];
550 spin_lock(&rd->grant_table->lock);
552 if ( frame == 0 )
553 {
554 frame = act->frame;
555 }
556 else if ( frame == GNTUNMAP_DEV_FROM_VIRT )
557 {
558 if ( !( flags & GNTMAP_device_map ) )
559 PIN_FAIL(unmap_out, GNTST_bad_dev_addr,
560 "Bad frame number: frame not mapped for dev access.\n");
561 frame = act->frame;
563 /* Frame will be unmapped for device access below if virt addr okay. */
564 }
565 else
566 {
567 if ( unlikely(frame != act->frame) )
568 PIN_FAIL(unmap_out, GNTST_general_error,
569 "Bad frame number doesn't match gntref.\n");
570 if ( flags & GNTMAP_device_map )
571 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
572 : GNTPIN_devw_inc;
574 map->ref_and_flags &= ~GNTMAP_device_map;
575 (void)__put_user(0, &uop->dev_bus_addr);
577 /* Frame is now unmapped for device access. */
578 }
580 if ( (addr != 0) &&
581 (flags & GNTMAP_host_map) &&
582 ((act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask)) > 0))
583 {
584 if ( flags & GNTMAP_contains_pte )
585 {
586 if ( (rc = clear_grant_pte_mapping(addr, frame, ld)) < 0 )
587 goto unmap_out;
588 }
589 else
590 {
591 if ( (rc = clear_grant_va_mapping(addr, frame)) < 0 )
592 goto unmap_out;
593 }
595 map->ref_and_flags &= ~GNTMAP_host_map;
597 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_hstr_inc
598 : GNTPIN_hstw_inc;
600 if ( frame == GNTUNMAP_DEV_FROM_VIRT )
601 {
602 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
603 : GNTPIN_devw_inc;
605 map->ref_and_flags &= ~GNTMAP_device_map;
606 (void)__put_user(0, &uop->dev_bus_addr);
607 }
609 rc = 0;
610 if ( !( flags & GNTMAP_contains_pte) )
611 *va = addr;
612 }
614 if ( (map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0)
615 {
616 map->ref_and_flags = 0;
617 put_maptrack_handle(ld->grant_table, handle);
618 }
620 /* If just unmapped a writable mapping, mark as dirtied */
621 if ( unlikely(shadow_mode_log_dirty(rd)) &&
622 !( flags & GNTMAP_readonly ) )
623 mark_dirty(rd, frame);
625 /* If the last writable mapping has been removed, put_page_type */
626 if ( ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask) ) == 0) &&
627 ( !( flags & GNTMAP_readonly ) ) )
628 {
629 clear_bit(_GTF_writing, &sha->flags);
630 put_page_type(&frame_table[frame]);
631 }
633 if ( act->pin == 0 )
634 {
635 act->frame = 0xdeadbeef;
636 clear_bit(_GTF_reading, &sha->flags);
637 put_page(&frame_table[frame]);
638 }
640 unmap_out:
641 (void)__put_user(rc, &uop->status);
642 spin_unlock(&rd->grant_table->lock);
643 put_domain(rd);
644 return rc;
645 }
647 static long
648 gnttab_unmap_grant_ref(
649 gnttab_unmap_grant_ref_t *uop, unsigned int count)
650 {
651 int i, flush = 0;
652 unsigned long va = 0;
654 for ( i = 0; i < count; i++ )
655 if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0 )
656 flush++;
658 if ( flush == 1 )
659 flush_tlb_one_mask(current->domain->cpumask, va);
660 else if ( flush != 0 )
661 flush_tlb_mask(current->domain->cpumask);
663 return 0;
664 }
666 static long
667 gnttab_setup_table(
668 gnttab_setup_table_t *uop, unsigned int count)
669 {
670 gnttab_setup_table_t op;
671 struct domain *d;
672 int i;
674 if ( count != 1 )
675 return -EINVAL;
677 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
678 {
679 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
680 return -EFAULT;
681 }
683 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
684 {
685 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
686 NR_GRANT_FRAMES);
687 (void)put_user(GNTST_general_error, &uop->status);
688 return 0;
689 }
691 if ( op.dom == DOMID_SELF )
692 {
693 op.dom = current->domain->domain_id;
694 }
695 else if ( unlikely(!IS_PRIV(current->domain)) )
696 {
697 (void)put_user(GNTST_permission_denied, &uop->status);
698 return 0;
699 }
701 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
702 {
703 DPRINTK("Bad domid %d.\n", op.dom);
704 (void)put_user(GNTST_bad_domain, &uop->status);
705 return 0;
706 }
708 if ( op.nr_frames <= NR_GRANT_FRAMES )
709 {
710 ASSERT(d->grant_table != NULL);
711 (void)put_user(GNTST_okay, &uop->status);
712 for ( i = 0; i < op.nr_frames; i++ )
713 (void)put_user(
714 (virt_to_phys(d->grant_table->shared) >> PAGE_SHIFT) + i,
715 &uop->frame_list[i]);
716 }
718 put_domain(d);
719 return 0;
720 }
722 #if GRANT_DEBUG
723 static int
724 gnttab_dump_table(gnttab_dump_table_t *uop)
725 {
726 grant_table_t *gt;
727 gnttab_dump_table_t op;
728 struct domain *d;
729 u32 shared_mfn;
730 active_grant_entry_t *act;
731 grant_entry_t sha_copy;
732 grant_mapping_t *maptrack;
733 int i;
736 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
737 {
738 DPRINTK("Fault while reading gnttab_dump_table_t.\n");
739 return -EFAULT;
740 }
742 if ( op.dom == DOMID_SELF )
743 {
744 op.dom = current->domain->domain_id;
745 }
747 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
748 {
749 DPRINTK("Bad domid %d.\n", op.dom);
750 (void)put_user(GNTST_bad_domain, &uop->status);
751 return 0;
752 }
754 ASSERT(d->grant_table != NULL);
755 gt = d->grant_table;
756 (void)put_user(GNTST_okay, &uop->status);
758 shared_mfn = virt_to_phys(d->grant_table->shared);
760 DPRINTK("Grant table for dom (%hu) MFN (%x)\n",
761 op.dom, shared_mfn);
763 ASSERT(d->grant_table->active != NULL);
764 ASSERT(d->grant_table->shared != NULL);
765 ASSERT(d->grant_table->maptrack != NULL);
767 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
768 {
769 sha_copy = gt->shared[i];
771 if ( sha_copy.flags )
772 {
773 DPRINTK("Grant: dom (%hu) SHARED (%d) flags:(%hx) "
774 "dom:(%hu) frame:(%x)\n",
775 op.dom, i, sha_copy.flags, sha_copy.domid, sha_copy.frame);
776 }
777 }
779 spin_lock(&gt->lock);
781 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
782 {
783 act = &gt->active[i];
785 if ( act->pin )
786 {
787 DPRINTK("Grant: dom (%hu) ACTIVE (%d) pin:(%x) "
788 "dom:(%hu) frame:(%lx)\n",
789 op.dom, i, act->pin, act->domid, act->frame);
790 }
791 }
793 for ( i = 0; i < gt->maptrack_limit; i++ )
794 {
795 maptrack = &gt->maptrack[i];
797 if ( maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK )
798 {
799 DPRINTK("Grant: dom (%hu) MAP (%d) ref:(%hu) flags:(%x) "
800 "dom:(%hu)\n",
801 op.dom, i,
802 maptrack->ref_and_flags >> MAPTRACK_REF_SHIFT,
803 maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK,
804 maptrack->domid);
805 }
806 }
808 spin_unlock(&gt->lock);
810 put_domain(d);
811 return 0;
812 }
813 #endif
815 static long
816 gnttab_donate(gnttab_donate_t *uop, unsigned int count)
817 {
818 struct domain *d = current->domain;
819 struct domain *e;
820 struct pfn_info *page;
821 u32 _d, _nd, x, y;
822 int i;
823 int result = GNTST_okay;
825 for (i = 0; i < count; i++) {
826 gnttab_donate_t *gop = &uop[i];
827 #if GRANT_DEBUG
828 printk("gnttab_donate: i=%d mfn=%lx domid=%d gref=%08x\n",
829 i, gop->mfn, gop->domid, gop->handle);
830 #endif
831 page = &frame_table[gop->mfn];
833 if (unlikely(IS_XEN_HEAP_FRAME(page))) {
834 printk("gnttab_donate: xen heap frame mfn=%lx\n",
835 (unsigned long) gop->mfn);
836 gop->status = GNTST_bad_virt_addr;
837 continue;
838 }
839 if (unlikely(!pfn_valid(page_to_pfn(page)))) {
840 printk("gnttab_donate: invalid pfn for mfn=%lx\n",
841 (unsigned long) gop->mfn);
842 gop->status = GNTST_bad_virt_addr;
843 continue;
844 }
845 if (unlikely((e = find_domain_by_id(gop->domid)) == NULL)) {
846 printk("gnttab_donate: can't find domain %d\n", gop->domid);
847 gop->status = GNTST_bad_domain;
848 continue;
849 }
851 spin_lock(&d->page_alloc_lock);
853 /*
854 * The tricky bit: atomically release ownership while
855 * there is just one benign reference to the page
856 * (PGC_allocated). If that reference disappears then the
857 * deallocation routine will safely spin.
858 */
859 _d = pickle_domptr(d);
860 _nd = page->u.inuse._domain;
861 y = page->count_info;
862 do {
863 x = y;
864 if (unlikely((x & (PGC_count_mask|PGC_allocated)) !=
865 (1 | PGC_allocated)) || unlikely(_nd != _d)) {
866 printk("gnttab_donate: Bad page values %p: ed=%p(%u), sd=%p,"
867 " caf=%08x, taf=%" PRtype_info "\n",
868 (void *) page_to_pfn(page),
869 d, d->domain_id, unpickle_domptr(_nd), x,
870 page->u.inuse.type_info);
871 spin_unlock(&d->page_alloc_lock);
872 put_domain(e);
873 return 0;
874 }
875 __asm__ __volatile__(
876 LOCK_PREFIX "cmpxchg8b %2"
877 : "=d" (_nd), "=a" (y),
878 "=m" (*(volatile u64 *)(&page->count_info))
879 : "0" (_d), "1" (x), "c" (NULL), "b" (x) );
880 } while (unlikely(_nd != _d) || unlikely(y != x));
882 /*
883 * Unlink from 'd'. At least one reference remains (now
884 * anonymous), so noone else is spinning to try to delete
885 * this page from 'd'.
886 */
887 d->tot_pages--;
888 list_del(&page->list);
890 spin_unlock(&d->page_alloc_lock);
892 spin_lock(&e->page_alloc_lock);
894 /*
895 * Check that 'e' will accept the page and has reservation
896 * headroom. Also, a domain mustn't have PGC_allocated
897 * pages when it is dying.
898 */
899 #ifdef GRANT_DEBUG
900 if (unlikely(e->tot_pages >= e->max_pages)) {
901 printk("gnttab_dontate: no headroom tot_pages=%d max_pages=%d\n",
902 e->tot_pages, e->max_pages);
903 spin_unlock(&e->page_alloc_lock);
904 put_domain(e);
905 result = GNTST_general_error;
906 break;
907 }
908 if (unlikely(test_bit(DOMFLAGS_DYING, &e->domain_flags))) {
909 printk("gnttab_donate: target domain is dying\n");
910 spin_unlock(&e->page_alloc_lock);
911 put_domain(e);
912 result = GNTST_general_error;
913 break;
914 }
915 if (unlikely(!gnttab_prepare_for_transfer(e, d, gop->handle))) {
916 printk("gnttab_donate: gnttab_prepare_for_transfer fails\n");
917 spin_unlock(&e->page_alloc_lock);
918 put_domain(e);
919 result = GNTST_general_error;
920 break;
921 }
922 #else
923 ASSERT(e->tot_pages <= e->max_pages);
924 if (unlikely(test_bit(DOMFLAGS_DYING, &e->domain_flags)) ||
925 unlikely(e->tot_pages == e->max_pages) ||
926 unlikely(!gnttab_prepare_for_transfer(e, d, gop->handle))) {
927 printk("gnttab_donate: Transferee has no reservation headroom (%d,"
928 "%d) or provided a bad grant ref (%08x) or is dying (%p)\n",
929 e->tot_pages, e->max_pages, gop->handle, e->d_flags);
930 spin_unlock(&e->page_alloc_lock);
931 put_domain(e);
932 result = GNTST_general_error;
933 break;
934 }
935 #endif
936 /* Okay, add the page to 'e'. */
937 if (unlikely(e->tot_pages++ == 0)) {
938 get_knownalive_domain(e);
939 }
940 list_add_tail(&page->list, &e->page_list);
941 page_set_owner(page, e);
943 spin_unlock(&e->page_alloc_lock);
945 /*
946 * Transfer is all done: tell the guest about its new page
947 * frame.
948 */
949 gnttab_notify_transfer(e, d, gop->handle, gop->mfn);
951 put_domain(e);
953 gop->status = GNTST_okay;
954 }
955 return result;
956 }
958 long
959 do_grant_table_op(
960 unsigned int cmd, void *uop, unsigned int count)
961 {
962 long rc;
963 struct domain *d = current->domain;
965 if ( count > 512 )
966 return -EINVAL;
968 LOCK_BIGLOCK(d);
970 sync_pagetable_state(d);
972 rc = -EFAULT;
973 switch ( cmd )
974 {
975 case GNTTABOP_map_grant_ref:
976 if ( unlikely(!array_access_ok(
977 uop, count, sizeof(gnttab_map_grant_ref_t))) )
978 goto out;
979 rc = gnttab_map_grant_ref((gnttab_map_grant_ref_t *)uop, count);
980 break;
981 case GNTTABOP_unmap_grant_ref:
982 if ( unlikely(!array_access_ok(
983 uop, count, sizeof(gnttab_unmap_grant_ref_t))) )
984 goto out;
985 rc = gnttab_unmap_grant_ref((gnttab_unmap_grant_ref_t *)uop,
986 count);
987 break;
988 case GNTTABOP_setup_table:
989 rc = gnttab_setup_table((gnttab_setup_table_t *)uop, count);
990 break;
991 #if GRANT_DEBUG
992 case GNTTABOP_dump_table:
993 rc = gnttab_dump_table((gnttab_dump_table_t *)uop);
994 break;
995 #endif
996 case GNTTABOP_donate:
997 if (unlikely(!array_access_ok(uop, count,
998 sizeof(gnttab_donate_t))))
999 goto out;
1000 rc = gnttab_donate(uop, count);
1001 break;
1002 default:
1003 rc = -ENOSYS;
1004 break;
1007 out:
1008 UNLOCK_BIGLOCK(d);
1010 return rc;
1013 int
1014 gnttab_check_unmap(
1015 struct domain *rd, struct domain *ld, unsigned long frame, int readonly)
1017 /* Called when put_page is invoked on a page belonging to a foreign domain.
1018 * Instead of decrementing the frame table ref count, locate the grant
1019 * table entry, if any, and if found, decrement that count.
1020 * Called a _lot_ at domain creation because pages mapped by priv domains
1021 * also traverse this.
1022 */
1024 /* Note: If the same frame is mapped multiple times, and then one of
1025 * the ptes is overwritten, which maptrack handle gets invalidated?
1026 * Advice: Don't do it. Explicitly unmap.
1027 */
1029 unsigned int handle, ref, refcount;
1030 grant_table_t *lgt, *rgt;
1031 active_grant_entry_t *act;
1032 grant_mapping_t *map;
1033 int found = 0;
1035 lgt = ld->grant_table;
1037 #if GRANT_DEBUG_VERBOSE
1038 if ( ld->domain_ id != 0 ) {
1039 DPRINTK("Foreign unref rd(%d) ld(%d) frm(%lx) flgs(%x).\n",
1040 rd->domain_id, ld->domain_id, frame, readonly);
1042 #endif
1044 /* Fast exit if we're not mapping anything using grant tables */
1045 if ( lgt->map_count == 0 )
1046 return 0;
1048 if ( get_domain(rd) == 0 ) {
1049 DPRINTK("gnttab_check_unmap: couldn't get_domain rd(%d)\n",
1050 rd->domain_id);
1051 return 0;
1054 rgt = rd->grant_table;
1056 for ( handle = 0; handle < lgt->maptrack_limit; handle++ ) {
1058 map = &lgt->maptrack[handle];
1060 if ( map->domid != rd->domain_id )
1061 continue;
1063 if ( ( map->ref_and_flags & MAPTRACK_GNTMAP_MASK ) &&
1064 ( readonly ? 1 : (!(map->ref_and_flags & GNTMAP_readonly)))) {
1066 ref = (map->ref_and_flags >> MAPTRACK_REF_SHIFT);
1067 act = &rgt->active[ref];
1069 spin_lock(&rgt->lock);
1071 if ( act->frame != frame ) {
1072 spin_unlock(&rgt->lock);
1073 continue;
1076 refcount = act->pin & ( readonly ? GNTPIN_hstr_mask
1077 : GNTPIN_hstw_mask );
1079 if ( refcount == 0 ) {
1080 spin_unlock(&rgt->lock);
1081 continue;
1084 /* gotcha */
1085 DPRINTK("Grant unref rd(%d) ld(%d) frm(%lx) flgs(%x).\n",
1086 rd->domain_id, ld->domain_id, frame, readonly);
1088 if ( readonly )
1089 act->pin -= GNTPIN_hstr_inc;
1090 else {
1091 act->pin -= GNTPIN_hstw_inc;
1093 /* any more granted writable mappings? */
1094 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 ) {
1095 clear_bit(_GTF_writing, &rgt->shared[ref].flags);
1096 put_page_type(&frame_table[frame]);
1100 if ( act->pin == 0 ) {
1101 clear_bit(_GTF_reading, &rgt->shared[ref].flags);
1102 put_page(&frame_table[frame]);
1105 spin_unlock(&rgt->lock);
1107 clear_bit(GNTMAP_host_map, &map->ref_and_flags);
1109 if ( !(map->ref_and_flags & GNTMAP_device_map) )
1110 put_maptrack_handle(lgt, handle);
1112 found = 1;
1113 break;
1116 put_domain(rd);
1118 return found;
1121 int
1122 gnttab_prepare_for_transfer(
1123 struct domain *rd, struct domain *ld, grant_ref_t ref)
1125 grant_table_t *rgt;
1126 grant_entry_t *sha;
1127 domid_t sdom;
1128 u16 sflags;
1129 u32 scombo, prev_scombo;
1130 int retries = 0;
1131 unsigned long target_pfn;
1133 #if GRANT_DEBUG_VERBOSE
1134 DPRINTK("gnttab_prepare_for_transfer rd(%hu) ld(%hu) ref(%hu).\n",
1135 rd->domain_id, ld->domain_id, ref);
1136 #endif
1138 if ( unlikely((rgt = rd->grant_table) == NULL) ||
1139 unlikely(ref >= NR_GRANT_ENTRIES) )
1141 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n",
1142 rd->domain_id, ref);
1143 return 0;
1146 spin_lock(&rgt->lock);
1148 sha = &rgt->shared[ref];
1150 sflags = sha->flags;
1151 sdom = sha->domid;
1153 for ( ; ; )
1155 target_pfn = sha->frame;
1157 if ( unlikely(target_pfn >= max_page ) )
1159 DPRINTK("Bad pfn (%lx)\n", target_pfn);
1160 goto fail;
1163 if ( unlikely(sflags != GTF_accept_transfer) ||
1164 unlikely(sdom != ld->domain_id) )
1166 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
1167 sflags, sdom, ld->domain_id);
1168 goto fail;
1171 /* Merge two 16-bit values into a 32-bit combined update. */
1172 /* NB. Endianness! */
1173 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
1175 /* NB. prev_scombo is updated in place to seen value. */
1176 if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo,
1177 prev_scombo | GTF_transfer_committed)) )
1179 DPRINTK("Fault while modifying shared flags and domid.\n");
1180 goto fail;
1183 /* Did the combined update work (did we see what we expected?). */
1184 if ( likely(prev_scombo == scombo) )
1185 break;
1187 if ( retries++ == 4 )
1189 DPRINTK("Shared grant entry is unstable.\n");
1190 goto fail;
1193 /* Didn't see what we expected. Split out the seen flags & dom. */
1194 /* NB. Endianness! */
1195 sflags = (u16)prev_scombo;
1196 sdom = (u16)(prev_scombo >> 16);
1199 spin_unlock(&rgt->lock);
1200 return 1;
1202 fail:
1203 spin_unlock(&rgt->lock);
1204 return 0;
1207 void
1208 gnttab_notify_transfer(
1209 struct domain *rd, struct domain *ld, grant_ref_t ref, unsigned long frame)
1211 grant_entry_t *sha;
1212 unsigned long pfn;
1214 #if GRANT_DEBUG_VERBOSE
1215 DPRINTK("gnttab_notify_transfer rd(%hu) ld(%hu) ref(%hu).\n",
1216 rd->domain_id, ld->domain_id, ref);
1217 #endif
1219 sha = &rd->grant_table->shared[ref];
1221 spin_lock(&rd->grant_table->lock);
1223 pfn = sha->frame;
1225 if ( unlikely(pfn >= max_page ) )
1226 DPRINTK("Bad pfn (%lx)\n", pfn);
1227 else
1229 machine_to_phys_mapping[frame] = pfn;
1231 if ( unlikely(shadow_mode_log_dirty(ld)))
1232 mark_dirty(ld, frame);
1234 if (shadow_mode_translate(ld))
1235 __phys_to_machine_mapping[pfn] = frame;
1237 sha->frame = __mfn_to_gpfn(rd, frame);
1238 sha->domid = rd->domain_id;
1239 wmb();
1240 sha->flags = ( GTF_accept_transfer | GTF_transfer_completed );
1242 spin_unlock(&rd->grant_table->lock);
1244 return;
1247 int
1248 grant_table_create(
1249 struct domain *d)
1251 grant_table_t *t;
1252 int i;
1254 if ( (t = xmalloc(grant_table_t)) == NULL )
1255 goto no_mem;
1257 /* Simple stuff. */
1258 memset(t, 0, sizeof(*t));
1259 spin_lock_init(&t->lock);
1261 /* Active grant table. */
1262 if ( (t->active = xmalloc_array(active_grant_entry_t, NR_GRANT_ENTRIES))
1263 == NULL )
1264 goto no_mem;
1265 memset(t->active, 0, sizeof(active_grant_entry_t) * NR_GRANT_ENTRIES);
1267 /* Tracking of mapped foreign frames table */
1268 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
1269 goto no_mem;
1270 t->maptrack_order = 0;
1271 t->maptrack_limit = PAGE_SIZE / sizeof(grant_mapping_t);
1272 memset(t->maptrack, 0, PAGE_SIZE);
1273 for ( i = 0; i < t->maptrack_limit; i++ )
1274 t->maptrack[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
1276 /* Shared grant table. */
1277 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1278 if ( t->shared == NULL )
1279 goto no_mem;
1280 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1282 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1284 SHARE_PFN_WITH_DOMAIN(
1285 virt_to_page((char *)(t->shared)+(i*PAGE_SIZE)), d);
1286 machine_to_phys_mapping[(virt_to_phys(t->shared) >> PAGE_SHIFT) + i] =
1287 INVALID_M2P_ENTRY;
1290 /* Okay, install the structure. */
1291 wmb(); /* avoid races with lock-free access to d->grant_table */
1292 d->grant_table = t;
1293 return 0;
1295 no_mem:
1296 if ( t != NULL )
1298 xfree(t->active);
1299 if ( t->maptrack != NULL )
1300 free_xenheap_page(t->maptrack);
1301 xfree(t);
1303 return -ENOMEM;
1306 void
1307 gnttab_release_dev_mappings(grant_table_t *gt)
1309 grant_mapping_t *map;
1310 domid_t dom;
1311 grant_ref_t ref;
1312 u16 handle;
1313 struct domain *ld, *rd;
1314 unsigned long frame;
1315 active_grant_entry_t *act;
1316 grant_entry_t *sha;
1318 ld = current->domain;
1320 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1322 map = &gt->maptrack[handle];
1324 if ( map->ref_and_flags & GNTMAP_device_map )
1326 dom = map->domid;
1327 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
1329 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1330 handle, ref,
1331 map->ref_and_flags & MAPTRACK_GNTMAP_MASK, dom);
1333 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
1334 unlikely(ld == rd) )
1336 if ( rd != NULL )
1337 put_domain(rd);
1339 printk(KERN_WARNING "Grant release: No dom%d\n", dom);
1340 continue;
1343 act = &rd->grant_table->active[ref];
1344 sha = &rd->grant_table->shared[ref];
1346 spin_lock(&rd->grant_table->lock);
1348 if ( act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask) )
1350 frame = act->frame;
1352 if ( ( (act->pin & GNTPIN_hstw_mask) == 0 ) &&
1353 ( (act->pin & GNTPIN_devw_mask) > 0 ) )
1355 clear_bit(_GTF_writing, &sha->flags);
1356 put_page_type(&frame_table[frame]);
1359 act->pin &= ~(GNTPIN_devw_mask | GNTPIN_devr_mask);
1361 if ( act->pin == 0 )
1363 clear_bit(_GTF_reading, &sha->flags);
1364 map->ref_and_flags = 0;
1365 put_page(&frame_table[frame]);
1367 else
1368 map->ref_and_flags &= ~GNTMAP_device_map;
1371 spin_unlock(&rd->grant_table->lock);
1373 put_domain(rd);
1379 void
1380 grant_table_destroy(
1381 struct domain *d)
1383 grant_table_t *t;
1385 if ( (t = d->grant_table) != NULL )
1387 /* Free memory relating to this grant table. */
1388 d->grant_table = NULL;
1389 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1390 free_xenheap_page(t->maptrack);
1391 xfree(t->active);
1392 xfree(t);
1396 void
1397 grant_table_init(
1398 void)
1400 /* Nothing. */
1403 /*
1404 * Local variables:
1405 * mode: C
1406 * c-set-style: "BSD"
1407 * c-basic-offset: 4
1408 * tab-width: 4
1409 * indent-tabs-mode: nil
1410 * End:
1411 */