direct-io.hg

view xen/common/grant_table.c @ 5517:10e9028c8e3d

bitkeeper revision 1.1718.1.10 (42b7b19aqOS_1M8I4pIOFjiTPYWV-g)

Merge bk://xenbits.xensource.com/xen-unstable.bk
into spot.cl.cam.ac.uk:C:/Documents and Settings/iap10/xen-unstable.bk
author iap10@spot.cl.cam.ac.uk
date Tue Jun 21 06:20:10 2005 +0000 (2005-06-21)
parents aa52b853c28b
children 649cd37aa1ab
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
25 #define GRANT_DEBUG 0
26 #define GRANT_DEBUG_VERBOSE 0
28 #include <xen/config.h>
29 #include <xen/lib.h>
30 #include <xen/sched.h>
31 #include <xen/shadow.h>
32 #include <xen/mm.h>
33 #include <acm/acm_hooks.h>
35 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
36 do { \
37 DPRINTK( _f, ## _a ); \
38 rc = (_rc); \
39 goto _lbl; \
40 } while ( 0 )
42 static inline int
43 get_maptrack_handle(
44 grant_table_t *t)
45 {
46 unsigned int h;
47 if ( unlikely((h = t->maptrack_head) == t->maptrack_limit) )
48 return -1;
49 t->maptrack_head = t->maptrack[h].ref_and_flags >> MAPTRACK_REF_SHIFT;
50 t->map_count++;
51 return h;
52 }
54 static inline void
55 put_maptrack_handle(
56 grant_table_t *t, int handle)
57 {
58 t->maptrack[handle].ref_and_flags = t->maptrack_head << MAPTRACK_REF_SHIFT;
59 t->maptrack_head = handle;
60 t->map_count--;
61 }
63 static int
64 __gnttab_activate_grant_ref(
65 struct domain *mapping_d, /* IN */
66 struct vcpu *mapping_ed,
67 struct domain *granting_d,
68 grant_ref_t ref,
69 u16 dev_hst_ro_flags,
70 unsigned long host_virt_addr,
71 unsigned long *pframe ) /* OUT */
72 {
73 domid_t sdom;
74 u16 sflags;
75 active_grant_entry_t *act;
76 grant_entry_t *sha;
77 s16 rc = 1;
78 unsigned long frame = 0;
79 int retries = 0;
81 /*
82 * Objectives of this function:
83 * . Make the record ( granting_d, ref ) active, if not already.
84 * . Update shared grant entry of owner, indicating frame is mapped.
85 * . Increment the owner act->pin reference counts.
86 * . get_page on shared frame if new mapping.
87 * . get_page_type if this is first RW mapping of frame.
88 * . Add PTE to virtual address space of mapping_d, if necessary.
89 * Returns:
90 * . -ve: error
91 * . 1: ok
92 * . 0: ok and TLB invalidate of host_virt_addr needed.
93 *
94 * On success, *pframe contains mfn.
95 */
97 /*
98 * We bound the number of times we retry CMPXCHG on memory locations that
99 * we share with a guest OS. The reason is that the guest can modify that
100 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
101 * could cause us to livelock. There are a few cases where it is valid for
102 * the guest to race our updates (e.g., to change the GTF_readonly flag),
103 * so we allow a few retries before failing.
104 */
106 act = &granting_d->grant_table->active[ref];
107 sha = &granting_d->grant_table->shared[ref];
109 spin_lock(&granting_d->grant_table->lock);
111 if ( act->pin == 0 )
112 {
113 /* CASE 1: Activating a previously inactive entry. */
115 sflags = sha->flags;
116 sdom = sha->domid;
118 for ( ; ; )
119 {
120 u32 scombo, prev_scombo, new_scombo;
122 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
123 unlikely(sdom != mapping_d->domain_id) )
124 PIN_FAIL(unlock_out, GNTST_general_error,
125 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
126 sflags, sdom, mapping_d->domain_id);
128 /* Merge two 16-bit values into a 32-bit combined update. */
129 /* NB. Endianness! */
130 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
132 new_scombo = scombo | GTF_reading;
133 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
134 {
135 new_scombo |= GTF_writing;
136 if ( unlikely(sflags & GTF_readonly) )
137 PIN_FAIL(unlock_out, GNTST_general_error,
138 "Attempt to write-pin a r/o grant entry.\n");
139 }
141 /* NB. prev_scombo is updated in place to seen value. */
142 if ( unlikely(cmpxchg_user((u32 *)&sha->flags,
143 prev_scombo,
144 new_scombo)) )
145 PIN_FAIL(unlock_out, GNTST_general_error,
146 "Fault while modifying shared flags and domid.\n");
148 /* Did the combined update work (did we see what we expected?). */
149 if ( likely(prev_scombo == scombo) )
150 break;
152 if ( retries++ == 4 )
153 PIN_FAIL(unlock_out, GNTST_general_error,
154 "Shared grant entry is unstable.\n");
156 /* Didn't see what we expected. Split out the seen flags & dom. */
157 /* NB. Endianness! */
158 sflags = (u16)prev_scombo;
159 sdom = (u16)(prev_scombo >> 16);
160 }
162 /* rmb(); */ /* not on x86 */
164 frame = __gpfn_to_mfn_foreign(granting_d, sha->frame);
166 if ( unlikely(!pfn_valid(frame)) ||
167 unlikely(!((dev_hst_ro_flags & GNTMAP_readonly) ?
168 get_page(&frame_table[frame], granting_d) :
169 get_page_and_type(&frame_table[frame], granting_d,
170 PGT_writable_page))) )
171 {
172 clear_bit(_GTF_writing, &sha->flags);
173 clear_bit(_GTF_reading, &sha->flags);
174 PIN_FAIL(unlock_out, GNTST_general_error,
175 "Could not pin the granted frame (%lx)!\n", frame);
176 }
178 if ( dev_hst_ro_flags & GNTMAP_device_map )
179 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
180 GNTPIN_devr_inc : GNTPIN_devw_inc;
181 if ( dev_hst_ro_flags & GNTMAP_host_map )
182 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
183 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
184 act->domid = sdom;
185 act->frame = frame;
186 }
187 else
188 {
189 /* CASE 2: Active modications to an already active entry. */
191 /*
192 * A cheesy check for possible pin-count overflow.
193 * A more accurate check cannot be done with a single comparison.
194 */
195 if ( (act->pin & 0x80808080U) != 0 )
196 PIN_FAIL(unlock_out, ENOSPC,
197 "Risk of counter overflow %08x\n", act->pin);
199 frame = act->frame;
201 if ( !(dev_hst_ro_flags & GNTMAP_readonly) &&
202 !((sflags = sha->flags) & GTF_writing) )
203 {
204 for ( ; ; )
205 {
206 u16 prev_sflags;
208 if ( unlikely(sflags & GTF_readonly) )
209 PIN_FAIL(unlock_out, GNTST_general_error,
210 "Attempt to write-pin a r/o grant entry.\n");
212 prev_sflags = sflags;
214 /* NB. prev_sflags is updated in place to seen value. */
215 if ( unlikely(cmpxchg_user(&sha->flags, prev_sflags,
216 prev_sflags | GTF_writing)) )
217 PIN_FAIL(unlock_out, GNTST_general_error,
218 "Fault while modifying shared flags.\n");
220 if ( likely(prev_sflags == sflags) )
221 break;
223 if ( retries++ == 4 )
224 PIN_FAIL(unlock_out, GNTST_general_error,
225 "Shared grant entry is unstable.\n");
227 sflags = prev_sflags;
228 }
230 if ( unlikely(!get_page_type(&frame_table[frame],
231 PGT_writable_page)) )
232 {
233 clear_bit(_GTF_writing, &sha->flags);
234 PIN_FAIL(unlock_out, GNTST_general_error,
235 "Attempt to write-pin a unwritable page.\n");
236 }
237 }
239 if ( dev_hst_ro_flags & GNTMAP_device_map )
240 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
241 GNTPIN_devr_inc : GNTPIN_devw_inc;
243 if ( dev_hst_ro_flags & GNTMAP_host_map )
244 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
245 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
246 }
248 /*
249 * At this point:
250 * act->pin updated to reflect mapping.
251 * sha->flags updated to indicate to granting domain mapping done.
252 * frame contains the mfn.
253 */
255 spin_unlock(&granting_d->grant_table->lock);
257 if ( (host_virt_addr != 0) && (dev_hst_ro_flags & GNTMAP_host_map) )
258 {
259 /* Write update into the pagetable. */
260 l1_pgentry_t pte;
261 pte = l1e_from_pfn(frame, _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_DIRTY);
262 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
263 l1e_add_flags(pte,_PAGE_RW);
264 rc = update_grant_va_mapping( host_virt_addr, pte,
265 mapping_d, mapping_ed );
267 /*
268 * IMPORTANT: (rc == 0) => must flush / invalidate entry in TLB.
269 * This is done in the outer gnttab_map_grant_ref.
270 */
272 if ( rc < 0 )
273 {
274 /* Failure: undo and abort. */
276 spin_lock(&granting_d->grant_table->lock);
278 if ( dev_hst_ro_flags & GNTMAP_readonly )
279 {
280 act->pin -= GNTPIN_hstr_inc;
281 }
282 else
283 {
284 act->pin -= GNTPIN_hstw_inc;
285 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
286 {
287 clear_bit(_GTF_writing, &sha->flags);
288 put_page_type(&frame_table[frame]);
289 }
290 }
292 if ( act->pin == 0 )
293 {
294 clear_bit(_GTF_reading, &sha->flags);
295 put_page(&frame_table[frame]);
296 }
298 spin_unlock(&granting_d->grant_table->lock);
299 }
301 }
303 *pframe = frame;
304 return rc;
306 unlock_out:
307 spin_unlock(&granting_d->grant_table->lock);
308 return rc;
309 }
311 /*
312 * Returns 0 if TLB flush / invalidate required by caller.
313 * va will indicate the address to be invalidated.
314 */
315 static int
316 __gnttab_map_grant_ref(
317 gnttab_map_grant_ref_t *uop,
318 unsigned long *va)
319 {
320 domid_t dom;
321 grant_ref_t ref;
322 struct domain *ld, *rd;
323 struct vcpu *led;
324 u16 dev_hst_ro_flags;
325 int handle;
326 unsigned long frame = 0, host_virt_addr;
327 int rc;
329 led = current;
330 ld = led->domain;
332 /* Bitwise-OR avoids short-circuiting which screws control flow. */
333 if ( unlikely(__get_user(dom, &uop->dom) |
334 __get_user(ref, &uop->ref) |
335 __get_user(host_virt_addr, &uop->host_virt_addr) |
336 __get_user(dev_hst_ro_flags, &uop->flags)) )
337 {
338 DPRINTK("Fault while reading gnttab_map_grant_ref_t.\n");
339 return -EFAULT; /* don't set status */
340 }
343 if ( ((host_virt_addr != 0) || (dev_hst_ro_flags & GNTMAP_host_map)) &&
344 unlikely(!__addr_ok(host_virt_addr)))
345 {
346 DPRINTK("Bad virtual address (%lx) or flags (%x).\n",
347 host_virt_addr, dev_hst_ro_flags);
348 (void)__put_user(GNTST_bad_virt_addr, &uop->handle);
349 return GNTST_bad_gntref;
350 }
352 if ( unlikely(ref >= NR_GRANT_ENTRIES) ||
353 unlikely((dev_hst_ro_flags &
354 (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
355 {
356 DPRINTK("Bad ref (%d) or flags (%x).\n", ref, dev_hst_ro_flags);
357 (void)__put_user(GNTST_bad_gntref, &uop->handle);
358 return GNTST_bad_gntref;
359 }
361 if (acm_pre_grant_map_ref(dom)) {
362 (void)__put_user(GNTST_permission_denied, &uop->handle);
363 return GNTST_permission_denied;
364 }
366 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
367 unlikely(ld == rd) )
368 {
369 if ( rd != NULL )
370 put_domain(rd);
371 DPRINTK("Could not find domain %d\n", dom);
372 (void)__put_user(GNTST_bad_domain, &uop->handle);
373 return GNTST_bad_domain;
374 }
376 /* Get a maptrack handle. */
377 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
378 {
379 int i;
380 grant_mapping_t *new_mt;
381 grant_table_t *lgt = ld->grant_table;
383 /* Grow the maptrack table. */
384 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
385 if ( new_mt == NULL )
386 {
387 put_domain(rd);
388 DPRINTK("No more map handles available\n");
389 (void)__put_user(GNTST_no_device_space, &uop->handle);
390 return GNTST_no_device_space;
391 }
393 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
394 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
395 new_mt[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
397 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
398 lgt->maptrack = new_mt;
399 lgt->maptrack_order += 1;
400 lgt->maptrack_limit <<= 1;
402 printk("Doubled maptrack size\n");
403 handle = get_maptrack_handle(ld->grant_table);
404 }
406 #if GRANT_DEBUG_VERBOSE
407 DPRINTK("Mapping grant ref (%hu) for domain (%hu) with flags (%x)\n",
408 ref, dom, dev_hst_ro_flags);
409 #endif
411 if ( 0 <= ( rc = __gnttab_activate_grant_ref( ld, led, rd, ref,
412 dev_hst_ro_flags,
413 host_virt_addr, &frame)))
414 {
415 /*
416 * Only make the maptrack live _after_ writing the pte, in case we
417 * overwrite the same frame number, causing a maptrack walk to find it
418 */
419 ld->grant_table->maptrack[handle].domid = dom;
421 ld->grant_table->maptrack[handle].ref_and_flags
422 = (ref << MAPTRACK_REF_SHIFT) |
423 (dev_hst_ro_flags & MAPTRACK_GNTMAP_MASK);
425 (void)__put_user(frame, &uop->dev_bus_addr);
427 if ( dev_hst_ro_flags & GNTMAP_host_map )
428 *va = host_virt_addr;
430 (void)__put_user(handle, &uop->handle);
431 }
432 else
433 {
434 (void)__put_user(rc, &uop->handle);
435 put_maptrack_handle(ld->grant_table, handle);
436 }
438 put_domain(rd);
439 return rc;
440 }
442 static long
443 gnttab_map_grant_ref(
444 gnttab_map_grant_ref_t *uop, unsigned int count)
445 {
446 int i, flush = 0;
447 unsigned long va = 0;
449 for ( i = 0; i < count; i++ )
450 if ( __gnttab_map_grant_ref(&uop[i], &va) == 0 )
451 flush++;
453 if ( flush == 1 )
454 flush_tlb_one_mask(current->domain->cpumask, va);
455 else if ( flush != 0 )
456 flush_tlb_mask(current->domain->cpumask);
458 return 0;
459 }
461 static int
462 __gnttab_unmap_grant_ref(
463 gnttab_unmap_grant_ref_t *uop,
464 unsigned long *va)
465 {
466 domid_t dom;
467 grant_ref_t ref;
468 u16 handle;
469 struct domain *ld, *rd;
471 active_grant_entry_t *act;
472 grant_entry_t *sha;
473 grant_mapping_t *map;
474 u16 flags;
475 s16 rc = 1;
476 unsigned long frame, virt;
478 ld = current->domain;
480 /* Bitwise-OR avoids short-circuiting which screws control flow. */
481 if ( unlikely(__get_user(virt, &uop->host_virt_addr) |
482 __get_user(frame, &uop->dev_bus_addr) |
483 __get_user(handle, &uop->handle)) )
484 {
485 DPRINTK("Fault while reading gnttab_unmap_grant_ref_t.\n");
486 return -EFAULT; /* don't set status */
487 }
489 map = &ld->grant_table->maptrack[handle];
491 if ( unlikely(handle >= ld->grant_table->maptrack_limit) ||
492 unlikely(!(map->ref_and_flags & MAPTRACK_GNTMAP_MASK)) )
493 {
494 DPRINTK("Bad handle (%d).\n", handle);
495 (void)__put_user(GNTST_bad_handle, &uop->status);
496 return GNTST_bad_handle;
497 }
499 dom = map->domid;
500 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
501 flags = map->ref_and_flags & MAPTRACK_GNTMAP_MASK;
503 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
504 unlikely(ld == rd) )
505 {
506 if ( rd != NULL )
507 put_domain(rd);
508 DPRINTK("Could not find domain %d\n", dom);
509 (void)__put_user(GNTST_bad_domain, &uop->status);
510 return GNTST_bad_domain;
511 }
513 #if GRANT_DEBUG_VERBOSE
514 DPRINTK("Unmapping grant ref (%hu) for domain (%hu) with handle (%hu)\n",
515 ref, dom, handle);
516 #endif
518 act = &rd->grant_table->active[ref];
519 sha = &rd->grant_table->shared[ref];
521 spin_lock(&rd->grant_table->lock);
523 if ( frame == 0 )
524 {
525 frame = act->frame;
526 }
527 else if ( frame == GNTUNMAP_DEV_FROM_VIRT )
528 {
529 if ( !( flags & GNTMAP_device_map ) )
530 PIN_FAIL(unmap_out, GNTST_bad_dev_addr,
531 "Bad frame number: frame not mapped for dev access.\n");
532 frame = act->frame;
534 /* Frame will be unmapped for device access below if virt addr okay. */
535 }
536 else
537 {
538 if ( unlikely(frame != act->frame) )
539 PIN_FAIL(unmap_out, GNTST_general_error,
540 "Bad frame number doesn't match gntref.\n");
541 if ( flags & GNTMAP_device_map )
542 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
543 : GNTPIN_devw_inc;
545 map->ref_and_flags &= ~GNTMAP_device_map;
546 (void)__put_user(0, &uop->dev_bus_addr);
548 /* Frame is now unmapped for device access. */
549 }
551 if ( (virt != 0) &&
552 (flags & GNTMAP_host_map) &&
553 ((act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask)) > 0))
554 {
555 l1_pgentry_t *pl1e;
556 unsigned long _ol1e;
558 pl1e = &linear_pg_table[l1_linear_offset(virt)];
560 if ( unlikely(__get_user(_ol1e, (unsigned long *)pl1e) != 0) )
561 {
562 DPRINTK("Could not find PTE entry for address %lx\n", virt);
563 rc = -EINVAL;
564 goto unmap_out;
565 }
567 /*
568 * Check that the virtual address supplied is actually mapped to
569 * act->frame.
570 */
571 if ( unlikely((_ol1e >> PAGE_SHIFT) != frame ))
572 {
573 DPRINTK("PTE entry %lx for address %lx doesn't match frame %lx\n",
574 _ol1e, virt, frame);
575 rc = -EINVAL;
576 goto unmap_out;
577 }
579 /* Delete pagetable entry. */
580 if ( unlikely(__put_user(0, (unsigned long *)pl1e)))
581 {
582 DPRINTK("Cannot delete PTE entry at %p for virtual address %lx\n",
583 pl1e, virt);
584 rc = -EINVAL;
585 goto unmap_out;
586 }
588 map->ref_and_flags &= ~GNTMAP_host_map;
590 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_hstr_inc
591 : GNTPIN_hstw_inc;
593 if ( frame == GNTUNMAP_DEV_FROM_VIRT )
594 {
595 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
596 : GNTPIN_devw_inc;
598 map->ref_and_flags &= ~GNTMAP_device_map;
599 (void)__put_user(0, &uop->dev_bus_addr);
600 }
602 rc = 0;
603 *va = virt;
604 }
606 if ( (map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0)
607 {
608 map->ref_and_flags = 0;
609 put_maptrack_handle(ld->grant_table, handle);
610 }
612 /* If just unmapped a writable mapping, mark as dirtied */
613 if ( unlikely(shadow_mode_log_dirty(rd)) &&
614 !( flags & GNTMAP_readonly ) )
615 mark_dirty(rd, frame);
617 /* If the last writable mapping has been removed, put_page_type */
618 if ( ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask) ) == 0) &&
619 ( !( flags & GNTMAP_readonly ) ) )
620 {
621 clear_bit(_GTF_writing, &sha->flags);
622 put_page_type(&frame_table[frame]);
623 }
625 if ( act->pin == 0 )
626 {
627 clear_bit(_GTF_reading, &sha->flags);
628 put_page(&frame_table[frame]);
629 }
631 unmap_out:
632 (void)__put_user(rc, &uop->status);
633 spin_unlock(&rd->grant_table->lock);
634 put_domain(rd);
635 return rc;
636 }
638 static long
639 gnttab_unmap_grant_ref(
640 gnttab_unmap_grant_ref_t *uop, unsigned int count)
641 {
642 int i, flush = 0;
643 unsigned long va = 0;
645 for ( i = 0; i < count; i++ )
646 if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0 )
647 flush++;
649 if ( flush == 1 )
650 flush_tlb_one_mask(current->domain->cpumask, va);
651 else if ( flush != 0 )
652 flush_tlb_mask(current->domain->cpumask);
654 return 0;
655 }
657 static long
658 gnttab_setup_table(
659 gnttab_setup_table_t *uop, unsigned int count)
660 {
661 gnttab_setup_table_t op;
662 struct domain *d;
663 int i;
665 if ( count != 1 )
666 return -EINVAL;
668 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
669 {
670 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
671 return -EFAULT;
672 }
674 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
675 {
676 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
677 NR_GRANT_FRAMES);
678 (void)put_user(GNTST_general_error, &uop->status);
679 return 0;
680 }
682 if ( op.dom == DOMID_SELF )
683 {
684 op.dom = current->domain->domain_id;
685 }
686 else if ( unlikely(!IS_PRIV(current->domain)) )
687 {
688 (void)put_user(GNTST_permission_denied, &uop->status);
689 return 0;
690 }
692 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
693 {
694 DPRINTK("Bad domid %d.\n", op.dom);
695 (void)put_user(GNTST_bad_domain, &uop->status);
696 return 0;
697 }
699 if ( op.nr_frames <= NR_GRANT_FRAMES )
700 {
701 ASSERT(d->grant_table != NULL);
702 (void)put_user(GNTST_okay, &uop->status);
703 for ( i = 0; i < op.nr_frames; i++ )
704 (void)put_user(
705 (virt_to_phys(d->grant_table->shared) >> PAGE_SHIFT) + i,
706 &uop->frame_list[i]);
707 }
709 put_domain(d);
710 return 0;
711 }
713 #if GRANT_DEBUG
714 static int
715 gnttab_dump_table(gnttab_dump_table_t *uop)
716 {
717 grant_table_t *gt;
718 gnttab_dump_table_t op;
719 struct domain *d;
720 u32 shared_mfn;
721 active_grant_entry_t *act;
722 grant_entry_t sha_copy;
723 grant_mapping_t *maptrack;
724 int i;
727 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
728 {
729 DPRINTK("Fault while reading gnttab_dump_table_t.\n");
730 return -EFAULT;
731 }
733 if ( op.dom == DOMID_SELF )
734 {
735 op.dom = current->domain->domain_id;
736 }
738 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
739 {
740 DPRINTK("Bad domid %d.\n", op.dom);
741 (void)put_user(GNTST_bad_domain, &uop->status);
742 return 0;
743 }
745 ASSERT(d->grant_table != NULL);
746 gt = d->grant_table;
747 (void)put_user(GNTST_okay, &uop->status);
749 shared_mfn = virt_to_phys(d->grant_table->shared);
751 DPRINTK("Grant table for dom (%hu) MFN (%x)\n",
752 op.dom, shared_mfn);
754 ASSERT(d->grant_table->active != NULL);
755 ASSERT(d->grant_table->shared != NULL);
756 ASSERT(d->grant_table->maptrack != NULL);
758 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
759 {
760 sha_copy = gt->shared[i];
762 if ( sha_copy.flags )
763 {
764 DPRINTK("Grant: dom (%hu) SHARED (%d) flags:(%hx) "
765 "dom:(%hu) frame:(%lx)\n",
766 op.dom, i, sha_copy.flags, sha_copy.domid, sha_copy.frame);
767 }
768 }
770 spin_lock(&gt->lock);
772 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
773 {
774 act = &gt->active[i];
776 if ( act->pin )
777 {
778 DPRINTK("Grant: dom (%hu) ACTIVE (%d) pin:(%x) "
779 "dom:(%hu) frame:(%lx)\n",
780 op.dom, i, act->pin, act->domid, act->frame);
781 }
782 }
784 for ( i = 0; i < gt->maptrack_limit; i++ )
785 {
786 maptrack = &gt->maptrack[i];
788 if ( maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK )
789 {
790 DPRINTK("Grant: dom (%hu) MAP (%d) ref:(%hu) flags:(%x) "
791 "dom:(%hu)\n",
792 op.dom, i,
793 maptrack->ref_and_flags >> MAPTRACK_REF_SHIFT,
794 maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK,
795 maptrack->domid);
796 }
797 }
799 spin_unlock(&gt->lock);
801 put_domain(d);
802 return 0;
803 }
804 #endif
806 long
807 do_grant_table_op(
808 unsigned int cmd, void *uop, unsigned int count)
809 {
810 long rc;
812 if ( count > 512 )
813 return -EINVAL;
815 LOCK_BIGLOCK(current->domain);
817 rc = -EFAULT;
818 switch ( cmd )
819 {
820 case GNTTABOP_map_grant_ref:
821 if ( unlikely(!array_access_ok(
822 uop, count, sizeof(gnttab_map_grant_ref_t))) )
823 goto out;
824 rc = gnttab_map_grant_ref((gnttab_map_grant_ref_t *)uop, count);
825 break;
826 case GNTTABOP_unmap_grant_ref:
827 if ( unlikely(!array_access_ok(
828 uop, count, sizeof(gnttab_unmap_grant_ref_t))) )
829 goto out;
830 rc = gnttab_unmap_grant_ref((gnttab_unmap_grant_ref_t *)uop, count);
831 break;
832 case GNTTABOP_setup_table:
833 rc = gnttab_setup_table((gnttab_setup_table_t *)uop, count);
834 break;
835 #if GRANT_DEBUG
836 case GNTTABOP_dump_table:
837 rc = gnttab_dump_table((gnttab_dump_table_t *)uop);
838 break;
839 #endif
840 default:
841 rc = -ENOSYS;
842 break;
843 }
845 out:
846 UNLOCK_BIGLOCK(current->domain);
848 return rc;
849 }
851 int
852 gnttab_check_unmap(
853 struct domain *rd, struct domain *ld, unsigned long frame, int readonly)
854 {
855 /* Called when put_page is invoked on a page belonging to a foreign domain.
856 * Instead of decrementing the frame table ref count, locate the grant
857 * table entry, if any, and if found, decrement that count.
858 * Called a _lot_ at domain creation because pages mapped by priv domains
859 * also traverse this.
860 */
862 /* Note: If the same frame is mapped multiple times, and then one of
863 * the ptes is overwritten, which maptrack handle gets invalidated?
864 * Advice: Don't do it. Explicitly unmap.
865 */
867 unsigned int handle, ref, refcount;
868 grant_table_t *lgt, *rgt;
869 active_grant_entry_t *act;
870 grant_mapping_t *map;
871 int found = 0;
873 lgt = ld->grant_table;
875 #if GRANT_DEBUG_VERBOSE
876 if ( ld->domain_id != 0 )
877 {
878 DPRINTK("Foreign unref rd(%d) ld(%d) frm(%x) flgs(%x).\n",
879 rd->domain_id, ld->domain_id, frame, readonly);
880 }
881 #endif
883 /* Fast exit if we're not mapping anything using grant tables */
884 if ( lgt->map_count == 0 )
885 return 0;
887 if ( get_domain(rd) == 0 )
888 {
889 DPRINTK("gnttab_check_unmap: couldn't get_domain rd(%d)\n",
890 rd->domain_id);
891 return 0;
892 }
894 rgt = rd->grant_table;
896 for ( handle = 0; handle < lgt->maptrack_limit; handle++ )
897 {
898 map = &lgt->maptrack[handle];
900 if ( ( map->ref_and_flags & MAPTRACK_GNTMAP_MASK ) &&
901 ( readonly ? 1 : (!(map->ref_and_flags & GNTMAP_readonly))))
902 {
903 ref = (map->ref_and_flags >> MAPTRACK_REF_SHIFT);
904 act = &rgt->active[ref];
906 spin_lock(&rgt->lock);
908 if ( act->frame != frame )
909 {
910 spin_unlock(&rgt->lock);
911 continue;
912 }
914 refcount = act->pin & ( readonly ? GNTPIN_hstr_mask
915 : GNTPIN_hstw_mask );
916 if ( refcount == 0 )
917 {
918 spin_unlock(&rgt->lock);
919 continue;
920 }
922 /* gotcha */
923 DPRINTK("Grant unref rd(%d) ld(%d) frm(%lx) flgs(%x).\n",
924 rd->domain_id, ld->domain_id, frame, readonly);
926 if ( readonly )
927 act->pin -= GNTPIN_hstr_inc;
928 else
929 {
930 act->pin -= GNTPIN_hstw_inc;
932 /* any more granted writable mappings? */
933 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
934 {
935 clear_bit(_GTF_writing, &rgt->shared[ref].flags);
936 put_page_type(&frame_table[frame]);
937 }
938 }
940 if ( act->pin == 0 )
941 {
942 clear_bit(_GTF_reading, &rgt->shared[ref].flags);
943 put_page(&frame_table[frame]);
944 }
945 spin_unlock(&rgt->lock);
947 clear_bit(GNTMAP_host_map, &map->ref_and_flags);
949 if ( !(map->ref_and_flags & GNTMAP_device_map) )
950 put_maptrack_handle(lgt, handle);
952 found = 1;
953 break;
954 }
955 }
956 put_domain(rd);
958 return found;
959 }
961 int
962 gnttab_prepare_for_transfer(
963 struct domain *rd, struct domain *ld, grant_ref_t ref)
964 {
965 grant_table_t *rgt;
966 grant_entry_t *sha;
967 domid_t sdom;
968 u16 sflags;
969 u32 scombo, prev_scombo;
970 int retries = 0;
971 unsigned long target_pfn;
973 DPRINTK("gnttab_prepare_for_transfer rd(%hu) ld(%hu) ref(%hu).\n",
974 rd->domain_id, ld->domain_id, ref);
976 if ( unlikely((rgt = rd->grant_table) == NULL) ||
977 unlikely(ref >= NR_GRANT_ENTRIES) )
978 {
979 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n",
980 rd->domain_id, ref);
981 return 0;
982 }
984 spin_lock(&rgt->lock);
986 sha = &rgt->shared[ref];
988 sflags = sha->flags;
989 sdom = sha->domid;
991 for ( ; ; )
992 {
993 target_pfn = sha->frame;
995 if ( unlikely(target_pfn >= max_page ) )
996 {
997 DPRINTK("Bad pfn (%lx)\n", target_pfn);
998 goto fail;
999 }
1001 if ( unlikely(sflags != GTF_accept_transfer) ||
1002 unlikely(sdom != ld->domain_id) )
1004 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
1005 sflags, sdom, ld->domain_id);
1006 goto fail;
1009 /* Merge two 16-bit values into a 32-bit combined update. */
1010 /* NB. Endianness! */
1011 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
1013 /* NB. prev_scombo is updated in place to seen value. */
1014 if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo,
1015 prev_scombo | GTF_transfer_committed)) )
1017 DPRINTK("Fault while modifying shared flags and domid.\n");
1018 goto fail;
1021 /* Did the combined update work (did we see what we expected?). */
1022 if ( likely(prev_scombo == scombo) )
1023 break;
1025 if ( retries++ == 4 )
1027 DPRINTK("Shared grant entry is unstable.\n");
1028 goto fail;
1031 /* Didn't see what we expected. Split out the seen flags & dom. */
1032 /* NB. Endianness! */
1033 sflags = (u16)prev_scombo;
1034 sdom = (u16)(prev_scombo >> 16);
1037 spin_unlock(&rgt->lock);
1038 return 1;
1040 fail:
1041 spin_unlock(&rgt->lock);
1042 return 0;
1045 void
1046 gnttab_notify_transfer(
1047 struct domain *rd, struct domain *ld, grant_ref_t ref, unsigned long frame)
1049 grant_entry_t *sha;
1050 unsigned long pfn;
1052 DPRINTK("gnttab_notify_transfer rd(%hu) ld(%hu) ref(%hu).\n",
1053 rd->domain_id, ld->domain_id, ref);
1055 sha = &rd->grant_table->shared[ref];
1057 spin_lock(&rd->grant_table->lock);
1059 pfn = sha->frame;
1061 if ( unlikely(pfn >= max_page ) )
1062 DPRINTK("Bad pfn (%lx)\n", pfn);
1063 else
1065 machine_to_phys_mapping[frame] = pfn;
1067 if ( unlikely(shadow_mode_log_dirty(ld)))
1068 mark_dirty(ld, frame);
1070 if (shadow_mode_translate(ld))
1071 __phys_to_machine_mapping[pfn] = frame;
1073 sha->frame = __mfn_to_gpfn(rd, frame);
1074 sha->domid = rd->domain_id;
1075 wmb();
1076 sha->flags = ( GTF_accept_transfer | GTF_transfer_completed );
1078 spin_unlock(&rd->grant_table->lock);
1080 return;
1083 int
1084 grant_table_create(
1085 struct domain *d)
1087 grant_table_t *t;
1088 int i;
1090 if ( (t = xmalloc(grant_table_t)) == NULL )
1091 goto no_mem;
1093 /* Simple stuff. */
1094 memset(t, 0, sizeof(*t));
1095 spin_lock_init(&t->lock);
1097 /* Active grant table. */
1098 if ( (t->active = xmalloc_array(active_grant_entry_t, NR_GRANT_ENTRIES))
1099 == NULL )
1100 goto no_mem;
1101 memset(t->active, 0, sizeof(active_grant_entry_t) * NR_GRANT_ENTRIES);
1103 /* Tracking of mapped foreign frames table */
1104 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
1105 goto no_mem;
1106 t->maptrack_order = 0;
1107 t->maptrack_limit = PAGE_SIZE / sizeof(grant_mapping_t);
1108 memset(t->maptrack, 0, PAGE_SIZE);
1109 for ( i = 0; i < t->maptrack_limit; i++ )
1110 t->maptrack[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
1112 /* Shared grant table. */
1113 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1114 if ( t->shared == NULL )
1115 goto no_mem;
1116 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1118 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1120 SHARE_PFN_WITH_DOMAIN(
1121 virt_to_page((char *)(t->shared)+(i*PAGE_SIZE)), d);
1122 machine_to_phys_mapping[(virt_to_phys(t->shared) >> PAGE_SHIFT) + i] =
1123 INVALID_M2P_ENTRY;
1126 /* Okay, install the structure. */
1127 wmb(); /* avoid races with lock-free access to d->grant_table */
1128 d->grant_table = t;
1129 return 0;
1131 no_mem:
1132 if ( t != NULL )
1134 xfree(t->active);
1135 if ( t->maptrack != NULL )
1136 free_xenheap_page(t->maptrack);
1137 xfree(t);
1139 return -ENOMEM;
1142 void
1143 gnttab_release_dev_mappings(grant_table_t *gt)
1145 grant_mapping_t *map;
1146 domid_t dom;
1147 grant_ref_t ref;
1148 u16 handle;
1149 struct domain *ld, *rd;
1150 unsigned long frame;
1151 active_grant_entry_t *act;
1152 grant_entry_t *sha;
1154 ld = current->domain;
1156 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1158 map = &gt->maptrack[handle];
1160 if ( map->ref_and_flags & GNTMAP_device_map )
1162 dom = map->domid;
1163 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
1165 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1166 handle, ref,
1167 map->ref_and_flags & MAPTRACK_GNTMAP_MASK, dom);
1169 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
1170 unlikely(ld == rd) )
1172 if ( rd != NULL )
1173 put_domain(rd);
1175 printk(KERN_WARNING "Grant release: No dom%d\n", dom);
1176 continue;
1179 act = &rd->grant_table->active[ref];
1180 sha = &rd->grant_table->shared[ref];
1182 spin_lock(&rd->grant_table->lock);
1184 if ( act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask) )
1186 frame = act->frame;
1188 if ( ( (act->pin & GNTPIN_hstw_mask) == 0 ) &&
1189 ( (act->pin & GNTPIN_devw_mask) > 0 ) )
1191 clear_bit(_GTF_writing, &sha->flags);
1192 put_page_type(&frame_table[frame]);
1195 act->pin &= ~(GNTPIN_devw_mask | GNTPIN_devr_mask);
1197 if ( act->pin == 0 )
1199 clear_bit(_GTF_reading, &sha->flags);
1200 map->ref_and_flags = 0;
1201 put_page(&frame_table[frame]);
1203 else
1204 map->ref_and_flags &= ~GNTMAP_device_map;
1207 spin_unlock(&rd->grant_table->lock);
1209 put_domain(rd);
1215 void
1216 grant_table_destroy(
1217 struct domain *d)
1219 grant_table_t *t;
1221 if ( (t = d->grant_table) != NULL )
1223 /* Free memory relating to this grant table. */
1224 d->grant_table = NULL;
1225 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1226 free_xenheap_page(t->maptrack);
1227 xfree(t->active);
1228 xfree(t);
1232 void
1233 grant_table_init(
1234 void)
1236 /* Nothing. */
1239 /*
1240 * Local variables:
1241 * mode: C
1242 * c-set-style: "BSD"
1243 * c-basic-offset: 4
1244 * tab-width: 4
1245 * indent-tabs-mode: nil
1246 * End:
1247 */