direct-io.hg

view xen/common/grant_table.c @ 8121:58d46463413e

GNTTABOP_map_grant_ref returns error status and handle as
separate fields. Update callers for new interface. Also
use int16_t as standard error code type on all public
interfaces.

Signed-off-by: Keir Fraser <keir@xensource.com>
author kaf24@firebug.cl.cam.ac.uk
date Wed Nov 30 17:24:27 2005 +0100 (2005-11-30)
parents b3f8d3158a1c
children 1e89d78f21c5
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/lib.h>
28 #include <xen/sched.h>
29 #include <xen/shadow.h>
30 #include <xen/mm.h>
31 #include <acm/acm_hooks.h>
32 #include <xen/trace.h>
34 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
35 do { \
36 DPRINTK( _f, ## _a ); \
37 rc = (_rc); \
38 goto _lbl; \
39 } while ( 0 )
41 static inline int
42 get_maptrack_handle(
43 grant_table_t *t)
44 {
45 unsigned int h;
46 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
47 return -1;
48 t->maptrack_head = t->maptrack[h].ref_and_flags >> MAPTRACK_REF_SHIFT;
49 t->map_count++;
50 return h;
51 }
53 static inline void
54 put_maptrack_handle(
55 grant_table_t *t, int handle)
56 {
57 t->maptrack[handle].ref_and_flags = t->maptrack_head << MAPTRACK_REF_SHIFT;
58 t->maptrack_head = handle;
59 t->map_count--;
60 }
62 /*
63 * Returns 0 if TLB flush / invalidate required by caller.
64 * va will indicate the address to be invalidated.
65 *
66 * addr is _either_ a host virtual address, or the address of the pte to
67 * update, as indicated by the GNTMAP_contains_pte flag.
68 */
69 static int
70 __gnttab_map_grant_ref(
71 gnttab_map_grant_ref_t *uop)
72 {
73 domid_t dom;
74 grant_ref_t ref;
75 struct domain *ld, *rd;
76 struct vcpu *led;
77 u16 dev_hst_ro_flags;
78 int handle;
79 u64 addr;
80 unsigned long frame = 0;
81 int rc = GNTST_okay;
82 active_grant_entry_t *act;
84 /* Entry details from @rd's shared grant table. */
85 grant_entry_t *sha;
86 domid_t sdom;
87 u16 sflags;
89 /*
90 * We bound the number of times we retry CMPXCHG on memory locations that
91 * we share with a guest OS. The reason is that the guest can modify that
92 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
93 * could cause us to livelock. There are a few cases where it is valid for
94 * the guest to race our updates (e.g., to change the GTF_readonly flag),
95 * so we allow a few retries before failing.
96 */
97 int retries = 0;
99 led = current;
100 ld = led->domain;
102 /* Bitwise-OR avoids short-circuiting which screws control flow. */
103 if ( unlikely(__get_user(dom, &uop->dom) |
104 __get_user(ref, &uop->ref) |
105 __get_user(addr, &uop->host_addr) |
106 __get_user(dev_hst_ro_flags, &uop->flags)) )
107 {
108 DPRINTK("Fault while reading gnttab_map_grant_ref_t.\n");
109 return -EFAULT; /* don't set status */
110 }
112 if ( unlikely(ref >= NR_GRANT_ENTRIES) ||
113 unlikely((dev_hst_ro_flags &
114 (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
115 {
116 DPRINTK("Bad ref (%d) or flags (%x).\n", ref, dev_hst_ro_flags);
117 (void)__put_user(GNTST_bad_gntref, &uop->status);
118 return GNTST_bad_gntref;
119 }
121 if ( acm_pre_grant_map_ref(dom) )
122 {
123 (void)__put_user(GNTST_permission_denied, &uop->status);
124 return GNTST_permission_denied;
125 }
127 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
128 unlikely(ld == rd) )
129 {
130 if ( rd != NULL )
131 put_domain(rd);
132 DPRINTK("Could not find domain %d\n", dom);
133 (void)__put_user(GNTST_bad_domain, &uop->status);
134 return GNTST_bad_domain;
135 }
137 /* Get a maptrack handle. */
138 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
139 {
140 int i;
141 grant_mapping_t *new_mt;
142 grant_table_t *lgt = ld->grant_table;
144 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
145 {
146 put_domain(rd);
147 DPRINTK("Maptrack table is at maximum size.\n");
148 (void)__put_user(GNTST_no_device_space, &uop->status);
149 return GNTST_no_device_space;
150 }
152 /* Grow the maptrack table. */
153 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
154 if ( new_mt == NULL )
155 {
156 put_domain(rd);
157 DPRINTK("No more map handles available.\n");
158 (void)__put_user(GNTST_no_device_space, &uop->status);
159 return GNTST_no_device_space;
160 }
162 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
163 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
164 new_mt[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
166 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
167 lgt->maptrack = new_mt;
168 lgt->maptrack_order += 1;
169 lgt->maptrack_limit <<= 1;
171 DPRINTK("Doubled maptrack size\n");
172 handle = get_maptrack_handle(ld->grant_table);
173 }
175 act = &rd->grant_table->active[ref];
176 sha = &rd->grant_table->shared[ref];
178 spin_lock(&rd->grant_table->lock);
180 if ( act->pin == 0 )
181 {
182 /* CASE 1: Activating a previously inactive entry. */
184 sflags = sha->flags;
185 sdom = sha->domid;
187 /* This loop attempts to set the access (reading/writing) flags
188 * in the grant table entry. It tries a cmpxchg on the field
189 * up to five times, and then fails under the assumption that
190 * the guest is misbehaving. */
191 for ( ; ; )
192 {
193 u32 scombo, prev_scombo, new_scombo;
195 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
196 unlikely(sdom != led->domain->domain_id) )
197 PIN_FAIL(unlock_out, GNTST_general_error,
198 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
199 sflags, sdom, led->domain->domain_id);
201 /* Merge two 16-bit values into a 32-bit combined update. */
202 /* NB. Endianness! */
203 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
205 new_scombo = scombo | GTF_reading;
206 if ( !(dev_hst_ro_flags & GNTMAP_readonly) )
207 {
208 new_scombo |= GTF_writing;
209 if ( unlikely(sflags & GTF_readonly) )
210 PIN_FAIL(unlock_out, GNTST_general_error,
211 "Attempt to write-pin a r/o grant entry.\n");
212 }
214 /* NB. prev_scombo is updated in place to seen value. */
215 if ( unlikely(cmpxchg_user((u32 *)&sha->flags,
216 prev_scombo,
217 new_scombo)) )
218 PIN_FAIL(unlock_out, GNTST_general_error,
219 "Fault while modifying shared flags and domid.\n");
221 /* Did the combined update work (did we see what we expected?). */
222 if ( likely(prev_scombo == scombo) )
223 break;
225 if ( retries++ == 4 )
226 PIN_FAIL(unlock_out, GNTST_general_error,
227 "Shared grant entry is unstable.\n");
229 /* Didn't see what we expected. Split out the seen flags & dom. */
230 /* NB. Endianness! */
231 sflags = (u16)prev_scombo;
232 sdom = (u16)(prev_scombo >> 16);
233 }
235 /* rmb(); */ /* not on x86 */
237 frame = __gpfn_to_mfn(rd, sha->frame);
239 if ( unlikely(!pfn_valid(frame)) ||
240 unlikely(!((dev_hst_ro_flags & GNTMAP_readonly) ?
241 get_page(&frame_table[frame], rd) :
242 get_page_and_type(&frame_table[frame], rd,
243 PGT_writable_page))) )
244 {
245 clear_bit(_GTF_writing, &sha->flags);
246 clear_bit(_GTF_reading, &sha->flags);
247 PIN_FAIL(unlock_out, GNTST_general_error,
248 "Could not pin the granted frame (%lx)!\n", frame);
249 }
251 if ( dev_hst_ro_flags & GNTMAP_device_map )
252 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
253 GNTPIN_devr_inc : GNTPIN_devw_inc;
254 if ( dev_hst_ro_flags & GNTMAP_host_map )
255 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
256 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
257 act->domid = sdom;
258 act->frame = frame;
259 }
260 else
261 {
262 /* CASE 2: Active modications to an already active entry. */
264 /*
265 * A cheesy check for possible pin-count overflow.
266 * A more accurate check cannot be done with a single comparison.
267 */
268 if ( (act->pin & 0x80808080U) != 0 )
269 PIN_FAIL(unlock_out, ENOSPC,
270 "Risk of counter overflow %08x\n", act->pin);
272 sflags = sha->flags;
273 frame = act->frame;
275 if ( !(dev_hst_ro_flags & GNTMAP_readonly) &&
276 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
277 {
278 for ( ; ; )
279 {
280 u16 prev_sflags;
282 if ( unlikely(sflags & GTF_readonly) )
283 PIN_FAIL(unlock_out, GNTST_general_error,
284 "Attempt to write-pin a r/o grant entry.\n");
286 prev_sflags = sflags;
288 /* NB. prev_sflags is updated in place to seen value. */
289 if ( unlikely(cmpxchg_user(&sha->flags, prev_sflags,
290 prev_sflags | GTF_writing)) )
291 PIN_FAIL(unlock_out, GNTST_general_error,
292 "Fault while modifying shared flags.\n");
294 if ( likely(prev_sflags == sflags) )
295 break;
297 if ( retries++ == 4 )
298 PIN_FAIL(unlock_out, GNTST_general_error,
299 "Shared grant entry is unstable.\n");
301 sflags = prev_sflags;
302 }
304 if ( unlikely(!get_page_type(&frame_table[frame],
305 PGT_writable_page)) )
306 {
307 clear_bit(_GTF_writing, &sha->flags);
308 PIN_FAIL(unlock_out, GNTST_general_error,
309 "Attempt to write-pin a unwritable page.\n");
310 }
311 }
313 if ( dev_hst_ro_flags & GNTMAP_device_map )
314 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
315 GNTPIN_devr_inc : GNTPIN_devw_inc;
317 if ( dev_hst_ro_flags & GNTMAP_host_map )
318 act->pin += (dev_hst_ro_flags & GNTMAP_readonly) ?
319 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
320 }
322 /*
323 * At this point:
324 * act->pin updated to reference count mappings.
325 * sha->flags updated to indicate to granting domain mapping done.
326 * frame contains the mfn.
327 */
329 spin_unlock(&rd->grant_table->lock);
331 if ( dev_hst_ro_flags & GNTMAP_host_map )
332 {
333 rc = create_grant_host_mapping(addr, frame, dev_hst_ro_flags);
334 if ( rc < 0 )
335 {
336 /* Failure: undo and abort. */
338 spin_lock(&rd->grant_table->lock);
340 if ( dev_hst_ro_flags & GNTMAP_readonly )
341 {
342 act->pin -= GNTPIN_hstr_inc;
343 }
344 else
345 {
346 act->pin -= GNTPIN_hstw_inc;
347 if ( (act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) == 0 )
348 {
349 clear_bit(_GTF_writing, &sha->flags);
350 put_page_type(&frame_table[frame]);
351 }
352 }
354 if ( act->pin == 0 )
355 {
356 clear_bit(_GTF_reading, &sha->flags);
357 put_page(&frame_table[frame]);
358 }
360 spin_unlock(&rd->grant_table->lock);
361 }
362 }
364 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, dom);
366 ld->grant_table->maptrack[handle].domid = dom;
367 ld->grant_table->maptrack[handle].ref_and_flags =
368 (ref << MAPTRACK_REF_SHIFT) |
369 (dev_hst_ro_flags & MAPTRACK_GNTMAP_MASK);
371 (void)__put_user((u64)frame << PAGE_SHIFT, &uop->dev_bus_addr);
372 (void)__put_user(handle, &uop->handle);
373 (void)__put_user(GNTST_okay, &uop->status);
375 put_domain(rd);
376 return rc;
379 unlock_out:
380 spin_unlock(&rd->grant_table->lock);
381 (void)__put_user(rc, &uop->status);
382 put_maptrack_handle(ld->grant_table, handle);
383 return rc;
384 }
386 static long
387 gnttab_map_grant_ref(
388 gnttab_map_grant_ref_t *uop, unsigned int count)
389 {
390 int i;
392 for ( i = 0; i < count; i++ )
393 (void)__gnttab_map_grant_ref(&uop[i]);
395 return 0;
396 }
398 static int
399 __gnttab_unmap_grant_ref(
400 gnttab_unmap_grant_ref_t *uop)
401 {
402 domid_t dom;
403 grant_ref_t ref;
404 grant_handle_t handle;
405 struct domain *ld, *rd;
406 active_grant_entry_t *act;
407 grant_entry_t *sha;
408 grant_mapping_t *map;
409 u16 flags;
410 s16 rc = 0;
411 u64 addr, dev_bus_addr;
412 unsigned long frame;
414 ld = current->domain;
416 /* Bitwise-OR avoids short-circuiting which screws control flow. */
417 if ( unlikely(__get_user(addr, &uop->host_addr) |
418 __get_user(dev_bus_addr, &uop->dev_bus_addr) |
419 __get_user(handle, &uop->handle)) )
420 {
421 DPRINTK("Fault while reading gnttab_unmap_grant_ref_t.\n");
422 return -EFAULT; /* don't set status */
423 }
425 frame = (unsigned long)(dev_bus_addr >> PAGE_SHIFT);
427 map = &ld->grant_table->maptrack[handle];
429 if ( unlikely(handle >= ld->grant_table->maptrack_limit) ||
430 unlikely(!(map->ref_and_flags & MAPTRACK_GNTMAP_MASK)) )
431 {
432 DPRINTK("Bad handle (%d).\n", handle);
433 (void)__put_user(GNTST_bad_handle, &uop->status);
434 return GNTST_bad_handle;
435 }
437 dom = map->domid;
438 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
439 flags = map->ref_and_flags & MAPTRACK_GNTMAP_MASK;
441 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) ||
442 unlikely(ld == rd) )
443 {
444 if ( rd != NULL )
445 put_domain(rd);
446 DPRINTK("Could not find domain %d\n", dom);
447 (void)__put_user(GNTST_bad_domain, &uop->status);
448 return GNTST_bad_domain;
449 }
451 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
453 act = &rd->grant_table->active[ref];
454 sha = &rd->grant_table->shared[ref];
456 spin_lock(&rd->grant_table->lock);
458 if ( frame == 0 )
459 {
460 frame = act->frame;
461 }
462 else
463 {
464 if ( unlikely(frame != act->frame) )
465 PIN_FAIL(unmap_out, GNTST_general_error,
466 "Bad frame number doesn't match gntref.\n");
467 if ( flags & GNTMAP_device_map )
468 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_devr_inc
469 : GNTPIN_devw_inc;
471 map->ref_and_flags &= ~GNTMAP_device_map;
472 (void)__put_user(0, &uop->dev_bus_addr);
473 }
475 if ( (addr != 0) &&
476 (flags & GNTMAP_host_map) &&
477 ((act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask)) > 0))
478 {
479 if ( (rc = destroy_grant_host_mapping(addr, frame, flags)) < 0 )
480 goto unmap_out;
482 map->ref_and_flags &= ~GNTMAP_host_map;
484 act->pin -= (flags & GNTMAP_readonly) ? GNTPIN_hstr_inc
485 : GNTPIN_hstw_inc;
486 }
488 if ( (map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0)
489 {
490 map->ref_and_flags = 0;
491 put_maptrack_handle(ld->grant_table, handle);
492 }
494 /* If just unmapped a writable mapping, mark as dirtied */
495 if ( !(flags & GNTMAP_readonly) )
496 gnttab_log_dirty(rd, frame);
498 /* If the last writable mapping has been removed, put_page_type */
499 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
500 !(flags & GNTMAP_readonly) )
501 {
502 clear_bit(_GTF_writing, &sha->flags);
503 put_page_type(&frame_table[frame]);
504 }
506 if ( act->pin == 0 )
507 {
508 act->frame = 0xdeadbeef;
509 clear_bit(_GTF_reading, &sha->flags);
510 put_page(&frame_table[frame]);
511 }
513 unmap_out:
514 (void)__put_user(rc, &uop->status);
515 spin_unlock(&rd->grant_table->lock);
516 put_domain(rd);
517 return rc;
518 }
520 static long
521 gnttab_unmap_grant_ref(
522 gnttab_unmap_grant_ref_t *uop, unsigned int count)
523 {
524 int i;
526 for ( i = 0; i < count; i++ )
527 (void)__gnttab_unmap_grant_ref(&uop[i]);
529 flush_tlb_mask(current->domain->cpumask);
531 return 0;
532 }
534 static long
535 gnttab_setup_table(
536 gnttab_setup_table_t *uop, unsigned int count)
537 {
538 gnttab_setup_table_t op;
539 struct domain *d;
540 int i;
542 if ( count != 1 )
543 return -EINVAL;
545 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
546 {
547 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
548 return -EFAULT;
549 }
551 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
552 {
553 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
554 NR_GRANT_FRAMES);
555 (void)put_user(GNTST_general_error, &uop->status);
556 return 0;
557 }
559 if ( op.dom == DOMID_SELF )
560 {
561 op.dom = current->domain->domain_id;
562 }
563 else if ( unlikely(!IS_PRIV(current->domain)) )
564 {
565 (void)put_user(GNTST_permission_denied, &uop->status);
566 return 0;
567 }
569 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
570 {
571 DPRINTK("Bad domid %d.\n", op.dom);
572 (void)put_user(GNTST_bad_domain, &uop->status);
573 return 0;
574 }
576 if ( op.nr_frames <= NR_GRANT_FRAMES )
577 {
578 ASSERT(d->grant_table != NULL);
579 (void)put_user(GNTST_okay, &uop->status);
580 for ( i = 0; i < op.nr_frames; i++ )
581 (void)put_user(gnttab_shared_mfn(d, d->grant_table, i),
582 &uop->frame_list[i]);
583 }
585 put_domain(d);
586 return 0;
587 }
589 static int
590 gnttab_dump_table(
591 gnttab_dump_table_t *uop)
592 {
593 grant_table_t *gt;
594 gnttab_dump_table_t op;
595 struct domain *d;
596 u32 shared_mfn;
597 active_grant_entry_t *act;
598 grant_entry_t sha_copy;
599 grant_mapping_t *maptrack;
600 int i;
602 if ( !IS_PRIV(current->domain) )
603 return -EPERM;
605 if ( unlikely(copy_from_user(&op, uop, sizeof(op)) != 0) )
606 {
607 DPRINTK("Fault while reading gnttab_dump_table_t.\n");
608 return -EFAULT;
609 }
611 if ( op.dom == DOMID_SELF )
612 op.dom = current->domain->domain_id;
614 if ( unlikely((d = find_domain_by_id(op.dom)) == NULL) )
615 {
616 DPRINTK("Bad domid %d.\n", op.dom);
617 (void)put_user(GNTST_bad_domain, &uop->status);
618 return 0;
619 }
621 ASSERT(d->grant_table != NULL);
622 gt = d->grant_table;
623 (void)put_user(GNTST_okay, &uop->status);
625 shared_mfn = virt_to_phys(d->grant_table->shared);
627 DPRINTK("Grant table for dom (%hu) MFN (%x)\n",
628 op.dom, shared_mfn);
630 ASSERT(d->grant_table->active != NULL);
631 ASSERT(d->grant_table->shared != NULL);
632 ASSERT(d->grant_table->maptrack != NULL);
634 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
635 {
636 sha_copy = gt->shared[i];
637 if ( sha_copy.flags )
638 DPRINTK("Grant: dom (%hu) SHARED (%d) flags:(%hx) "
639 "dom:(%hu) frame:(%x)\n",
640 op.dom, i, sha_copy.flags, sha_copy.domid, sha_copy.frame);
641 }
643 spin_lock(&gt->lock);
645 for ( i = 0; i < NR_GRANT_ENTRIES; i++ )
646 {
647 act = &gt->active[i];
648 if ( act->pin )
649 DPRINTK("Grant: dom (%hu) ACTIVE (%d) pin:(%x) "
650 "dom:(%hu) frame:(%lx)\n",
651 op.dom, i, act->pin, act->domid, act->frame);
652 }
654 for ( i = 0; i < gt->maptrack_limit; i++ )
655 {
656 maptrack = &gt->maptrack[i];
657 if ( maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK )
658 DPRINTK("Grant: dom (%hu) MAP (%d) ref:(%hu) flags:(%x) "
659 "dom:(%hu)\n",
660 op.dom, i,
661 maptrack->ref_and_flags >> MAPTRACK_REF_SHIFT,
662 maptrack->ref_and_flags & MAPTRACK_GNTMAP_MASK,
663 maptrack->domid);
664 }
666 spin_unlock(&gt->lock);
668 put_domain(d);
669 return 0;
670 }
672 static long
673 gnttab_transfer(
674 gnttab_transfer_t *uop, unsigned int count)
675 {
676 struct domain *d = current->domain;
677 struct domain *e;
678 struct pfn_info *page;
679 int i;
680 grant_entry_t *sha;
681 gnttab_transfer_t gop;
683 for ( i = 0; i < count; i++ )
684 {
685 /* Read from caller address space. */
686 if ( unlikely(__copy_from_user(&gop, &uop[i], sizeof(gop))) )
687 {
688 DPRINTK("gnttab_transfer: error reading req %d/%d\n", i, count);
689 (void)__put_user(GNTST_bad_page, &uop[i].status);
690 return -EFAULT; /* This is *very* fatal. */
691 }
693 /* Check the passed page frame for basic validity. */
694 page = &frame_table[gop.mfn];
695 if ( unlikely(!pfn_valid(gop.mfn) || IS_XEN_HEAP_FRAME(page)) )
696 {
697 DPRINTK("gnttab_transfer: out-of-range or xen frame %lx\n",
698 (unsigned long)gop.mfn);
699 (void)__put_user(GNTST_bad_page, &uop[i].status);
700 continue;
701 }
703 if ( steal_page_for_grant_transfer(d, page) < 0 )
704 {
705 (void)__put_user(GNTST_bad_page, &uop[i].status);
706 continue;
707 }
709 /* Find the target domain. */
710 if ( unlikely((e = find_domain_by_id(gop.domid)) == NULL) )
711 {
712 DPRINTK("gnttab_transfer: can't find domain %d\n", gop.domid);
713 (void)__put_user(GNTST_bad_domain, &uop[i].status);
714 page->count_info &= ~(PGC_count_mask|PGC_allocated);
715 free_domheap_page(page);
716 continue;
717 }
719 spin_lock(&e->page_alloc_lock);
721 /*
722 * Check that 'e' will accept the page and has reservation
723 * headroom. Also, a domain mustn't have PGC_allocated
724 * pages when it is dying.
725 */
726 if ( unlikely(test_bit(_DOMF_dying, &e->domain_flags)) ||
727 unlikely(e->tot_pages >= e->max_pages) ||
728 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
729 {
730 if ( !test_bit(_DOMF_dying, &e->domain_flags) )
731 DPRINTK("gnttab_transfer: Transferee has no reservation "
732 "headroom (%d,%d) or provided a bad grant ref (%08x) "
733 "or is dying (%lx)\n",
734 e->tot_pages, e->max_pages, gop.ref, e->domain_flags);
735 spin_unlock(&e->page_alloc_lock);
736 put_domain(e);
737 (void)__put_user(GNTST_general_error, &uop[i].status);
738 page->count_info &= ~(PGC_count_mask|PGC_allocated);
739 free_domheap_page(page);
740 continue;
741 }
743 /* Okay, add the page to 'e'. */
744 if ( unlikely(e->tot_pages++ == 0) )
745 get_knownalive_domain(e);
746 list_add_tail(&page->list, &e->page_list);
747 page_set_owner(page, e);
749 spin_unlock(&e->page_alloc_lock);
751 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
753 /* Tell the guest about its new page frame. */
754 sha = &e->grant_table->shared[gop.ref];
755 sha->frame = gop.mfn;
756 wmb();
757 sha->flags |= GTF_transfer_completed;
759 put_domain(e);
761 (void)__put_user(GNTST_okay, &uop[i].status);
762 }
764 return 0;
765 }
767 long
768 do_grant_table_op(
769 unsigned int cmd, void *uop, unsigned int count)
770 {
771 long rc;
772 struct domain *d = current->domain;
774 if ( count > 512 )
775 return -EINVAL;
777 LOCK_BIGLOCK(d);
779 sync_pagetable_state(d);
781 rc = -EFAULT;
782 switch ( cmd )
783 {
784 case GNTTABOP_map_grant_ref:
785 if ( unlikely(!array_access_ok(
786 uop, count, sizeof(gnttab_map_grant_ref_t))) )
787 goto out;
788 rc = gnttab_map_grant_ref((gnttab_map_grant_ref_t *)uop, count);
789 break;
790 case GNTTABOP_unmap_grant_ref:
791 if ( unlikely(!array_access_ok(
792 uop, count, sizeof(gnttab_unmap_grant_ref_t))) )
793 goto out;
794 rc = gnttab_unmap_grant_ref(
795 (gnttab_unmap_grant_ref_t *)uop, count);
796 break;
797 case GNTTABOP_setup_table:
798 rc = gnttab_setup_table((gnttab_setup_table_t *)uop, count);
799 break;
800 case GNTTABOP_dump_table:
801 rc = gnttab_dump_table((gnttab_dump_table_t *)uop);
802 break;
803 case GNTTABOP_transfer:
804 if (unlikely(!array_access_ok(
805 uop, count, sizeof(gnttab_transfer_t))))
806 goto out;
807 rc = gnttab_transfer(uop, count);
808 break;
809 default:
810 rc = -ENOSYS;
811 break;
812 }
814 out:
815 UNLOCK_BIGLOCK(d);
817 return rc;
818 }
820 int
821 gnttab_prepare_for_transfer(
822 struct domain *rd, struct domain *ld, grant_ref_t ref)
823 {
824 grant_table_t *rgt;
825 grant_entry_t *sha;
826 domid_t sdom;
827 u16 sflags;
828 u32 scombo, prev_scombo;
829 int retries = 0;
830 unsigned long target_pfn;
832 if ( unlikely((rgt = rd->grant_table) == NULL) ||
833 unlikely(ref >= NR_GRANT_ENTRIES) )
834 {
835 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n",
836 rd->domain_id, ref);
837 return 0;
838 }
840 spin_lock(&rgt->lock);
842 sha = &rgt->shared[ref];
844 sflags = sha->flags;
845 sdom = sha->domid;
847 for ( ; ; )
848 {
849 target_pfn = sha->frame;
851 if ( unlikely(target_pfn >= max_page ) )
852 {
853 DPRINTK("Bad pfn (%lx)\n", target_pfn);
854 goto fail;
855 }
857 if ( unlikely(sflags != GTF_accept_transfer) ||
858 unlikely(sdom != ld->domain_id) )
859 {
860 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
861 sflags, sdom, ld->domain_id);
862 goto fail;
863 }
865 /* Merge two 16-bit values into a 32-bit combined update. */
866 /* NB. Endianness! */
867 prev_scombo = scombo = ((u32)sdom << 16) | (u32)sflags;
869 /* NB. prev_scombo is updated in place to seen value. */
870 if ( unlikely(cmpxchg_user((u32 *)&sha->flags, prev_scombo,
871 prev_scombo | GTF_transfer_committed)) )
872 {
873 DPRINTK("Fault while modifying shared flags and domid.\n");
874 goto fail;
875 }
877 /* Did the combined update work (did we see what we expected?). */
878 if ( likely(prev_scombo == scombo) )
879 break;
881 if ( retries++ == 4 )
882 {
883 DPRINTK("Shared grant entry is unstable.\n");
884 goto fail;
885 }
887 /* Didn't see what we expected. Split out the seen flags & dom. */
888 /* NB. Endianness! */
889 sflags = (u16)prev_scombo;
890 sdom = (u16)(prev_scombo >> 16);
891 }
893 spin_unlock(&rgt->lock);
894 return 1;
896 fail:
897 spin_unlock(&rgt->lock);
898 return 0;
899 }
901 int
902 grant_table_create(
903 struct domain *d)
904 {
905 grant_table_t *t;
906 int i;
908 if ( (t = xmalloc(grant_table_t)) == NULL )
909 goto no_mem;
911 /* Simple stuff. */
912 memset(t, 0, sizeof(*t));
913 spin_lock_init(&t->lock);
915 /* Active grant table. */
916 if ( (t->active = xmalloc_array(active_grant_entry_t, NR_GRANT_ENTRIES))
917 == NULL )
918 goto no_mem;
919 memset(t->active, 0, sizeof(active_grant_entry_t) * NR_GRANT_ENTRIES);
921 /* Tracking of mapped foreign frames table */
922 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
923 goto no_mem;
924 t->maptrack_order = 0;
925 t->maptrack_limit = PAGE_SIZE / sizeof(grant_mapping_t);
926 memset(t->maptrack, 0, PAGE_SIZE);
927 for ( i = 0; i < t->maptrack_limit; i++ )
928 t->maptrack[i].ref_and_flags = (i+1) << MAPTRACK_REF_SHIFT;
930 /* Shared grant table. */
931 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
932 if ( t->shared == NULL )
933 goto no_mem;
934 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
936 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
937 gnttab_create_shared_mfn(d, t, i);
939 /* Okay, install the structure. */
940 wmb(); /* avoid races with lock-free access to d->grant_table */
941 d->grant_table = t;
942 return 0;
944 no_mem:
945 if ( t != NULL )
946 {
947 xfree(t->active);
948 free_xenheap_page(t->maptrack);
949 xfree(t);
950 }
951 return -ENOMEM;
952 }
954 void
955 gnttab_release_mappings(
956 struct domain *d)
957 {
958 grant_table_t *gt = d->grant_table;
959 grant_mapping_t *map;
960 grant_ref_t ref;
961 grant_handle_t handle;
962 struct domain *rd;
963 active_grant_entry_t *act;
964 grant_entry_t *sha;
966 BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
968 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
969 {
970 map = &gt->maptrack[handle];
971 if ( !(map->ref_and_flags & (GNTMAP_device_map|GNTMAP_host_map)) )
972 continue;
974 ref = map->ref_and_flags >> MAPTRACK_REF_SHIFT;
976 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
977 handle, ref, map->ref_and_flags & MAPTRACK_GNTMAP_MASK,
978 map->domid);
980 rd = find_domain_by_id(map->domid);
981 BUG_ON(rd == NULL);
983 spin_lock(&rd->grant_table->lock);
985 act = &rd->grant_table->active[ref];
986 sha = &rd->grant_table->shared[ref];
988 if ( map->ref_and_flags & GNTMAP_readonly )
989 {
990 if ( map->ref_and_flags & GNTMAP_device_map )
991 {
992 BUG_ON((act->pin & GNTPIN_devr_mask) == 0);
993 act->pin -= GNTPIN_devr_inc;
994 }
996 if ( map->ref_and_flags & GNTMAP_host_map )
997 {
998 BUG_ON((act->pin & GNTPIN_hstr_mask) == 0);
999 act->pin -= GNTPIN_hstr_inc;
1002 else
1004 if ( map->ref_and_flags & GNTMAP_device_map )
1006 BUG_ON((act->pin & GNTPIN_devw_mask) == 0);
1007 act->pin -= GNTPIN_devw_inc;
1010 if ( map->ref_and_flags & GNTMAP_host_map )
1012 BUG_ON((act->pin & GNTPIN_hstw_mask) == 0);
1013 act->pin -= GNTPIN_hstw_inc;
1016 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
1018 clear_bit(_GTF_writing, &sha->flags);
1019 put_page_type(&frame_table[act->frame]);
1023 if ( act->pin == 0 )
1025 clear_bit(_GTF_reading, &sha->flags);
1026 put_page(&frame_table[act->frame]);
1029 spin_unlock(&rd->grant_table->lock);
1031 put_domain(rd);
1033 map->ref_and_flags = 0;
1038 void
1039 grant_table_destroy(
1040 struct domain *d)
1042 grant_table_t *t;
1044 if ( (t = d->grant_table) != NULL )
1046 /* Free memory relating to this grant table. */
1047 d->grant_table = NULL;
1048 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1049 free_xenheap_page(t->maptrack);
1050 xfree(t->active);
1051 xfree(t);
1055 void
1056 grant_table_init(
1057 void)
1059 /* Nothing. */
1062 /*
1063 * Local variables:
1064 * mode: C
1065 * c-set-style: "BSD"
1066 * c-basic-offset: 4
1067 * tab-width: 4
1068 * indent-tabs-mode: nil
1069 * End:
1070 */