ia64/xen-unstable

view xen/common/grant_table.c @ 11439:383bc7c7b19e

[XEN] gnttab: Read domid/flags atomically.

This more neatly matches how the two adjacent fields in
the grant-table entry are accessed by the cmpxchg()
function.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
author kfraser@ubuntu.eng.hq.xensource.com
date Tue Sep 05 18:36:23 2006 -0700 (2006-09-05)
parents 66dd34f2f439
children 1de184deaa9c
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/lib.h>
28 #include <xen/sched.h>
29 #include <xen/shadow.h>
30 #include <xen/mm.h>
31 #include <xen/trace.h>
32 #include <xen/guest_access.h>
33 #include <xen/domain_page.h>
34 #include <acm/acm_hooks.h>
36 /*
37 * The first two members of a grant entry are updated as a combined pair.
38 * The following union allows that to happen in an endian-neutral fashion.
39 */
40 union grant_combo {
41 uint32_t word;
42 struct {
43 uint16_t flags;
44 domid_t domid;
45 } shorts;
46 };
48 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
49 do { \
50 DPRINTK( _f, ## _a ); \
51 rc = (_rc); \
52 goto _lbl; \
53 } while ( 0 )
55 static inline int
56 get_maptrack_handle(
57 struct grant_table *t)
58 {
59 unsigned int h;
60 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
61 return -1;
62 t->maptrack_head = t->maptrack[h].ref;
63 t->map_count++;
64 return h;
65 }
67 static inline void
68 put_maptrack_handle(
69 struct grant_table *t, int handle)
70 {
71 t->maptrack[handle].ref = t->maptrack_head;
72 t->maptrack_head = handle;
73 t->map_count--;
74 }
76 /*
77 * Returns 0 if TLB flush / invalidate required by caller.
78 * va will indicate the address to be invalidated.
79 *
80 * addr is _either_ a host virtual address, or the address of the pte to
81 * update, as indicated by the GNTMAP_contains_pte flag.
82 */
83 static void
84 __gnttab_map_grant_ref(
85 struct gnttab_map_grant_ref *op)
86 {
87 struct domain *ld, *rd;
88 struct vcpu *led;
89 int handle;
90 unsigned long frame = 0;
91 int rc = GNTST_okay;
92 struct active_grant_entry *act;
93 grant_entry_t *sha;
94 union grant_combo scombo, prev_scombo, new_scombo;
96 /*
97 * We bound the number of times we retry CMPXCHG on memory locations that
98 * we share with a guest OS. The reason is that the guest can modify that
99 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
100 * could cause us to livelock. There are a few cases where it is valid for
101 * the guest to race our updates (e.g., to change the GTF_readonly flag),
102 * so we allow a few retries before failing.
103 */
104 int retries = 0;
106 led = current;
107 ld = led->domain;
109 if ( unlikely(op->ref >= NR_GRANT_ENTRIES) ||
110 unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
111 {
112 DPRINTK("Bad ref (%d) or flags (%x).\n", op->ref, op->flags);
113 op->status = GNTST_bad_gntref;
114 return;
115 }
117 if ( acm_pre_grant_map_ref(op->dom) )
118 {
119 op->status = GNTST_permission_denied;
120 return;
121 }
123 if ( unlikely((rd = find_domain_by_id(op->dom)) == NULL) )
124 {
125 if ( rd != NULL )
126 put_domain(rd);
127 DPRINTK("Could not find domain %d\n", op->dom);
128 op->status = GNTST_bad_domain;
129 return;
130 }
132 /* Get a maptrack handle. */
133 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
134 {
135 int i;
136 struct grant_mapping *new_mt;
137 struct grant_table *lgt = ld->grant_table;
139 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
140 {
141 put_domain(rd);
142 DPRINTK("Maptrack table is at maximum size.\n");
143 op->status = GNTST_no_device_space;
144 return;
145 }
147 /* Grow the maptrack table. */
148 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
149 if ( new_mt == NULL )
150 {
151 put_domain(rd);
152 DPRINTK("No more map handles available.\n");
153 op->status = GNTST_no_device_space;
154 return;
155 }
157 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
158 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
159 new_mt[i].ref = i+1;
161 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
162 lgt->maptrack = new_mt;
163 lgt->maptrack_order += 1;
164 lgt->maptrack_limit <<= 1;
166 DPRINTK("Doubled maptrack size\n");
167 handle = get_maptrack_handle(ld->grant_table);
168 }
170 act = &rd->grant_table->active[op->ref];
171 sha = &rd->grant_table->shared[op->ref];
173 spin_lock(&rd->grant_table->lock);
175 /* If already pinned, check the active domid and avoid refcnt overflow. */
176 if ( act->pin &&
177 ((act->domid != ld->domain_id) ||
178 (act->pin & 0x80808080U) != 0) )
179 PIN_FAIL(unlock_out, GNTST_general_error,
180 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
181 act->domid, ld->domain_id, act->pin);
183 if ( !act->pin ||
184 (!(op->flags & GNTMAP_readonly) &&
185 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
186 {
187 scombo.word = *(u32 *)&sha->flags;
189 /*
190 * This loop attempts to set the access (reading/writing) flags
191 * in the grant table entry. It tries a cmpxchg on the field
192 * up to five times, and then fails under the assumption that
193 * the guest is misbehaving.
194 */
195 for ( ; ; )
196 {
197 /* If not already pinned, check the grant domid and type. */
198 if ( !act->pin &&
199 (((scombo.shorts.flags & GTF_type_mask) !=
200 GTF_permit_access) ||
201 (scombo.shorts.domid != ld->domain_id)) )
202 PIN_FAIL(unlock_out, GNTST_general_error,
203 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
204 scombo.shorts.flags, scombo.shorts.domid,
205 ld->domain_id);
207 new_scombo = scombo;
208 new_scombo.shorts.flags |= GTF_reading;
210 if ( !(op->flags & GNTMAP_readonly) )
211 {
212 new_scombo.shorts.flags |= GTF_writing;
213 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
214 PIN_FAIL(unlock_out, GNTST_general_error,
215 "Attempt to write-pin a r/o grant entry.\n");
216 }
218 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
219 scombo.word, new_scombo.word);
220 if ( likely(prev_scombo.word == scombo.word) )
221 break;
223 if ( retries++ == 4 )
224 PIN_FAIL(unlock_out, GNTST_general_error,
225 "Shared grant entry is unstable.\n");
227 scombo = prev_scombo;
228 }
230 if ( !act->pin )
231 {
232 act->domid = scombo.shorts.domid;
233 act->frame = gmfn_to_mfn(rd, sha->frame);
234 }
235 }
237 if ( op->flags & GNTMAP_device_map )
238 act->pin += (op->flags & GNTMAP_readonly) ?
239 GNTPIN_devr_inc : GNTPIN_devw_inc;
240 if ( op->flags & GNTMAP_host_map )
241 act->pin += (op->flags & GNTMAP_readonly) ?
242 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
244 spin_unlock(&rd->grant_table->lock);
246 frame = act->frame;
247 if ( unlikely(!mfn_valid(frame)) ||
248 unlikely(!((op->flags & GNTMAP_readonly) ?
249 get_page(mfn_to_page(frame), rd) :
250 get_page_and_type(mfn_to_page(frame), rd,
251 PGT_writable_page))) )
252 PIN_FAIL(undo_out, GNTST_general_error,
253 "Could not pin the granted frame (%lx)!\n", frame);
255 if ( op->flags & GNTMAP_host_map )
256 {
257 rc = create_grant_host_mapping(op->host_addr, frame, op->flags);
258 if ( rc != GNTST_okay )
259 {
260 if ( !(op->flags & GNTMAP_readonly) )
261 put_page_type(mfn_to_page(frame));
262 put_page(mfn_to_page(frame));
263 goto undo_out;
264 }
266 if ( op->flags & GNTMAP_device_map )
267 {
268 (void)get_page(mfn_to_page(frame), rd);
269 if ( !(op->flags & GNTMAP_readonly) )
270 get_page_type(mfn_to_page(frame), PGT_writable_page);
271 }
272 }
274 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
276 ld->grant_table->maptrack[handle].domid = op->dom;
277 ld->grant_table->maptrack[handle].ref = op->ref;
278 ld->grant_table->maptrack[handle].flags = op->flags;
280 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
281 op->handle = handle;
282 op->status = GNTST_okay;
284 put_domain(rd);
285 return;
287 undo_out:
288 spin_lock(&rd->grant_table->lock);
290 if ( op->flags & GNTMAP_device_map )
291 act->pin -= (op->flags & GNTMAP_readonly) ?
292 GNTPIN_devr_inc : GNTPIN_devw_inc;
293 if ( op->flags & GNTMAP_host_map )
294 act->pin -= (op->flags & GNTMAP_readonly) ?
295 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
297 if ( !(op->flags & GNTMAP_readonly) &&
298 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
299 gnttab_clear_flag(_GTF_writing, &sha->flags);
301 if ( !act->pin )
302 gnttab_clear_flag(_GTF_reading, &sha->flags);
304 unlock_out:
305 spin_unlock(&rd->grant_table->lock);
306 op->status = rc;
307 put_maptrack_handle(ld->grant_table, handle);
308 put_domain(rd);
309 }
311 static long
312 gnttab_map_grant_ref(
313 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
314 {
315 int i;
316 struct gnttab_map_grant_ref op;
318 for ( i = 0; i < count; i++ )
319 {
320 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
321 return -EFAULT;
322 __gnttab_map_grant_ref(&op);
323 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
324 return -EFAULT;
325 }
327 return 0;
328 }
330 static void
331 __gnttab_unmap_grant_ref(
332 struct gnttab_unmap_grant_ref *op)
333 {
334 domid_t dom;
335 grant_ref_t ref;
336 struct domain *ld, *rd;
337 struct active_grant_entry *act;
338 grant_entry_t *sha;
339 struct grant_mapping *map;
340 u16 flags;
341 s16 rc = 0;
342 unsigned long frame;
344 ld = current->domain;
346 frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
348 map = &ld->grant_table->maptrack[op->handle];
350 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) ||
351 unlikely(!map->flags) )
352 {
353 DPRINTK("Bad handle (%d).\n", op->handle);
354 op->status = GNTST_bad_handle;
355 return;
356 }
358 dom = map->domid;
359 ref = map->ref;
360 flags = map->flags;
362 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) )
363 {
364 if ( rd != NULL )
365 put_domain(rd);
366 DPRINTK("Could not find domain %d\n", dom);
367 op->status = GNTST_bad_domain;
368 return;
369 }
371 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
373 act = &rd->grant_table->active[ref];
374 sha = &rd->grant_table->shared[ref];
376 spin_lock(&rd->grant_table->lock);
378 if ( frame == 0 )
379 {
380 frame = act->frame;
381 }
382 else
383 {
384 if ( unlikely(frame != act->frame) )
385 PIN_FAIL(unmap_out, GNTST_general_error,
386 "Bad frame number doesn't match gntref.\n");
387 if ( flags & GNTMAP_device_map )
388 {
389 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
390 map->flags &= ~GNTMAP_device_map;
391 if ( flags & GNTMAP_readonly )
392 {
393 act->pin -= GNTPIN_devr_inc;
394 put_page(mfn_to_page(frame));
395 }
396 else
397 {
398 act->pin -= GNTPIN_devw_inc;
399 put_page_and_type(mfn_to_page(frame));
400 }
401 }
402 }
404 if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
405 {
406 if ( (rc = destroy_grant_host_mapping(op->host_addr,
407 frame, flags)) < 0 )
408 goto unmap_out;
410 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
411 map->flags &= ~GNTMAP_host_map;
412 if ( flags & GNTMAP_readonly )
413 {
414 act->pin -= GNTPIN_hstr_inc;
415 put_page(mfn_to_page(frame));
416 }
417 else
418 {
419 act->pin -= GNTPIN_hstw_inc;
420 put_page_and_type(mfn_to_page(frame));
421 }
422 }
424 if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
425 {
426 map->flags = 0;
427 put_maptrack_handle(ld->grant_table, op->handle);
428 }
430 /* If just unmapped a writable mapping, mark as dirtied */
431 if ( !(flags & GNTMAP_readonly) )
432 gnttab_mark_dirty(rd, frame);
434 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
435 !(flags & GNTMAP_readonly) )
436 gnttab_clear_flag(_GTF_writing, &sha->flags);
438 if ( act->pin == 0 )
439 gnttab_clear_flag(_GTF_reading, &sha->flags);
441 unmap_out:
442 op->status = rc;
443 spin_unlock(&rd->grant_table->lock);
444 put_domain(rd);
445 }
447 static long
448 gnttab_unmap_grant_ref(
449 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
450 {
451 int i;
452 struct gnttab_unmap_grant_ref op;
454 for ( i = 0; i < count; i++ )
455 {
456 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
457 goto fault;
458 __gnttab_unmap_grant_ref(&op);
459 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
460 goto fault;
461 }
463 flush_tlb_mask(current->domain->domain_dirty_cpumask);
464 return 0;
466 fault:
467 flush_tlb_mask(current->domain->domain_dirty_cpumask);
468 return -EFAULT;
469 }
471 static long
472 gnttab_setup_table(
473 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
474 {
475 struct gnttab_setup_table op;
476 struct domain *d;
477 int i;
478 unsigned long gmfn;
479 domid_t dom;
481 if ( count != 1 )
482 return -EINVAL;
484 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
485 {
486 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
487 return -EFAULT;
488 }
490 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
491 {
492 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
493 NR_GRANT_FRAMES);
494 op.status = GNTST_general_error;
495 goto out;
496 }
498 dom = op.dom;
499 if ( dom == DOMID_SELF )
500 {
501 dom = current->domain->domain_id;
502 }
503 else if ( unlikely(!IS_PRIV(current->domain)) )
504 {
505 op.status = GNTST_permission_denied;
506 goto out;
507 }
509 if ( unlikely((d = find_domain_by_id(dom)) == NULL) )
510 {
511 DPRINTK("Bad domid %d.\n", dom);
512 op.status = GNTST_bad_domain;
513 goto out;
514 }
516 ASSERT(d->grant_table != NULL);
517 op.status = GNTST_okay;
518 for ( i = 0; i < op.nr_frames; i++ )
519 {
520 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
521 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
522 }
524 put_domain(d);
526 out:
527 if ( unlikely(copy_to_guest(uop, &op, 1)) )
528 return -EFAULT;
530 return 0;
531 }
533 /*
534 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
535 * ownership of a page frame. If so, lock down the grant entry.
536 */
537 static int
538 gnttab_prepare_for_transfer(
539 struct domain *rd, struct domain *ld, grant_ref_t ref)
540 {
541 struct grant_table *rgt;
542 struct grant_entry *sha;
543 union grant_combo scombo, prev_scombo, new_scombo;
544 int retries = 0;
546 if ( unlikely((rgt = rd->grant_table) == NULL) ||
547 unlikely(ref >= NR_GRANT_ENTRIES) )
548 {
549 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n",
550 rd->domain_id, ref);
551 return 0;
552 }
554 spin_lock(&rgt->lock);
556 sha = &rgt->shared[ref];
558 scombo.word = *(u32 *)&sha->flags;
560 for ( ; ; )
561 {
562 if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) ||
563 unlikely(scombo.shorts.domid != ld->domain_id) )
564 {
565 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
566 scombo.shorts.flags, scombo.shorts.domid,
567 ld->domain_id);
568 goto fail;
569 }
571 new_scombo = scombo;
572 new_scombo.shorts.flags |= GTF_transfer_committed;
574 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
575 scombo.word, new_scombo.word);
576 if ( likely(prev_scombo.word == scombo.word) )
577 break;
579 if ( retries++ == 4 )
580 {
581 DPRINTK("Shared grant entry is unstable.\n");
582 goto fail;
583 }
585 scombo = prev_scombo;
586 }
588 spin_unlock(&rgt->lock);
589 return 1;
591 fail:
592 spin_unlock(&rgt->lock);
593 return 0;
594 }
596 static long
597 gnttab_transfer(
598 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
599 {
600 struct domain *d = current->domain;
601 struct domain *e;
602 struct page_info *page;
603 int i;
604 grant_entry_t *sha;
605 struct gnttab_transfer gop;
606 unsigned long mfn;
608 for ( i = 0; i < count; i++ )
609 {
610 /* Read from caller address space. */
611 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
612 {
613 DPRINTK("gnttab_transfer: error reading req %d/%d\n", i, count);
614 return -EFAULT;
615 }
617 mfn = gmfn_to_mfn(d, gop.mfn);
619 /* Check the passed page frame for basic validity. */
620 if ( unlikely(!mfn_valid(mfn)) )
621 {
622 DPRINTK("gnttab_transfer: out-of-range %lx\n",
623 (unsigned long)gop.mfn);
624 gop.status = GNTST_bad_page;
625 goto copyback;
626 }
628 page = mfn_to_page(mfn);
629 if ( unlikely(IS_XEN_HEAP_FRAME(page)) )
630 {
631 DPRINTK("gnttab_transfer: xen frame %lx\n",
632 (unsigned long)gop.mfn);
633 gop.status = GNTST_bad_page;
634 goto copyback;
635 }
637 if ( steal_page(d, page, 0) < 0 )
638 {
639 gop.status = GNTST_bad_page;
640 goto copyback;
641 }
643 /* Find the target domain. */
644 if ( unlikely((e = find_domain_by_id(gop.domid)) == NULL) )
645 {
646 DPRINTK("gnttab_transfer: can't find domain %d\n", gop.domid);
647 page->count_info &= ~(PGC_count_mask|PGC_allocated);
648 free_domheap_page(page);
649 gop.status = GNTST_bad_domain;
650 goto copyback;
651 }
653 spin_lock(&e->page_alloc_lock);
655 /*
656 * Check that 'e' will accept the page and has reservation
657 * headroom. Also, a domain mustn't have PGC_allocated
658 * pages when it is dying.
659 */
660 if ( unlikely(test_bit(_DOMF_dying, &e->domain_flags)) ||
661 unlikely(e->tot_pages >= e->max_pages) ||
662 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
663 {
664 if ( !test_bit(_DOMF_dying, &e->domain_flags) )
665 DPRINTK("gnttab_transfer: Transferee has no reservation "
666 "headroom (%d,%d) or provided a bad grant ref (%08x) "
667 "or is dying (%lx)\n",
668 e->tot_pages, e->max_pages, gop.ref, e->domain_flags);
669 spin_unlock(&e->page_alloc_lock);
670 put_domain(e);
671 page->count_info &= ~(PGC_count_mask|PGC_allocated);
672 free_domheap_page(page);
673 gop.status = GNTST_general_error;
674 goto copyback;
675 }
677 /* Okay, add the page to 'e'. */
678 if ( unlikely(e->tot_pages++ == 0) )
679 get_knownalive_domain(e);
680 list_add_tail(&page->list, &e->page_list);
681 page_set_owner(page, e);
683 spin_unlock(&e->page_alloc_lock);
685 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
687 /* Tell the guest about its new page frame. */
688 sha = &e->grant_table->shared[gop.ref];
689 guest_physmap_add_page(e, sha->frame, mfn);
690 sha->frame = mfn;
691 wmb();
692 sha->flags |= GTF_transfer_completed;
694 put_domain(e);
696 gop.status = GNTST_okay;
698 copyback:
699 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
700 {
701 DPRINTK("gnttab_transfer: error writing resp %d/%d\n", i, count);
702 return -EFAULT;
703 }
704 }
706 return 0;
707 }
709 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
710 type and reference counts. */
711 static void
712 __release_grant_for_copy(
713 struct domain *rd, unsigned long gref, int readonly)
714 {
715 grant_entry_t *const sha = &rd->grant_table->shared[gref];
716 struct active_grant_entry *const act = &rd->grant_table->active[gref];
717 const unsigned long r_frame = act->frame;
719 if ( !readonly )
720 gnttab_mark_dirty(rd, r_frame);
722 spin_lock(&rd->grant_table->lock);
724 if ( readonly )
725 {
726 act->pin -= GNTPIN_hstr_inc;
727 }
728 else
729 {
730 act->pin -= GNTPIN_hstw_inc;
731 if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) )
732 gnttab_clear_flag(_GTF_writing, &sha->flags);
733 }
735 if ( !act->pin )
736 gnttab_clear_flag(_GTF_reading, &sha->flags);
738 spin_unlock(&rd->grant_table->lock);
739 }
741 /* Grab a frame number from a grant entry and update the flags and pin
742 count as appropriate. Note that this does *not* update the page
743 type or reference counts. */
744 static int
745 __acquire_grant_for_copy(
746 struct domain *rd, unsigned long gref, int readonly,
747 unsigned long *frame)
748 {
749 grant_entry_t *sha;
750 struct active_grant_entry *act;
751 s16 rc = GNTST_okay;
752 int retries = 0;
753 union grant_combo scombo, prev_scombo, new_scombo;
755 if ( unlikely(gref >= NR_GRANT_ENTRIES) )
756 PIN_FAIL(error_out, GNTST_bad_gntref,
757 "Bad grant reference %ld\n", gref);
759 act = &rd->grant_table->active[gref];
760 sha = &rd->grant_table->shared[gref];
762 spin_lock(&rd->grant_table->lock);
764 /* If already pinned, check the active domid and avoid refcnt overflow. */
765 if ( act->pin &&
766 ((act->domid != current->domain->domain_id) ||
767 (act->pin & 0x80808080U) != 0) )
768 PIN_FAIL(unlock_out, GNTST_general_error,
769 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
770 act->domid, current->domain->domain_id, act->pin);
772 if ( !act->pin ||
773 (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) )
774 {
775 scombo.word = *(u32 *)&sha->flags;
777 for ( ; ; )
778 {
779 /* If not already pinned, check the grant domid and type. */
780 if ( !act->pin &&
781 (((scombo.shorts.flags & GTF_type_mask) !=
782 GTF_permit_access) ||
783 (scombo.shorts.domid != current->domain->domain_id)) )
784 PIN_FAIL(unlock_out, GNTST_general_error,
785 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
786 scombo.shorts.flags, scombo.shorts.domid,
787 current->domain->domain_id);
789 new_scombo = scombo;
790 new_scombo.shorts.flags |= GTF_reading;
792 if ( !readonly )
793 {
794 new_scombo.shorts.flags |= GTF_writing;
795 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
796 PIN_FAIL(unlock_out, GNTST_general_error,
797 "Attempt to write-pin a r/o grant entry.\n");
798 }
800 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
801 scombo.word, new_scombo.word);
802 if ( likely(prev_scombo.word == scombo.word) )
803 break;
805 if ( retries++ == 4 )
806 PIN_FAIL(unlock_out, GNTST_general_error,
807 "Shared grant entry is unstable.\n");
809 scombo = prev_scombo;
810 }
812 if ( !act->pin )
813 {
814 act->domid = scombo.shorts.domid;
815 act->frame = gmfn_to_mfn(rd, sha->frame);
816 }
817 }
819 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
821 *frame = act->frame;
823 unlock_out:
824 spin_unlock(&rd->grant_table->lock);
825 error_out:
826 return rc;
827 }
829 static void
830 __gnttab_copy(
831 struct gnttab_copy *op)
832 {
833 struct domain *sd = NULL, *dd = NULL;
834 unsigned long s_frame, d_frame;
835 char *sp, *dp;
836 s16 rc = GNTST_okay;
837 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
838 int src_is_gref, dest_is_gref;
840 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
841 ((op->dest.offset + op->len) > PAGE_SIZE) )
842 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
844 src_is_gref = op->flags & GNTCOPY_source_gref;
845 dest_is_gref = op->flags & GNTCOPY_dest_gref;
847 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
848 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
849 PIN_FAIL(error_out, GNTST_permission_denied,
850 "only allow copy-by-mfn for DOMID_SELF.\n");
852 if ( op->source.domid == DOMID_SELF )
853 {
854 sd = current->domain;
855 get_knownalive_domain(sd);
856 }
857 else if ( (sd = find_domain_by_id(op->source.domid)) == NULL )
858 {
859 PIN_FAIL(error_out, GNTST_bad_domain,
860 "couldn't find %d\n", op->source.domid);
861 }
863 if ( op->dest.domid == DOMID_SELF )
864 {
865 dd = current->domain;
866 get_knownalive_domain(dd);
867 }
868 else if ( (dd = find_domain_by_id(op->dest.domid)) == NULL )
869 {
870 PIN_FAIL(error_out, GNTST_bad_domain,
871 "couldn't find %d\n", op->dest.domid);
872 }
874 if ( src_is_gref )
875 {
876 rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame);
877 if ( rc != GNTST_okay )
878 goto error_out;
879 have_s_grant = 1;
880 }
881 else
882 {
883 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
884 }
885 if ( !get_page(mfn_to_page(s_frame), sd) )
886 PIN_FAIL(error_out, GNTST_general_error,
887 "could not get source frame %lx.\n", s_frame);
888 have_s_ref = 1;
890 if ( dest_is_gref )
891 {
892 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame);
893 if ( rc != GNTST_okay )
894 goto error_out;
895 have_d_grant = 1;
896 }
897 else
898 {
899 d_frame = gmfn_to_mfn(sd, op->dest.u.gmfn);
900 }
901 if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) )
902 PIN_FAIL(error_out, GNTST_general_error,
903 "could not get source frame %lx.\n", d_frame);
905 sp = map_domain_page(s_frame);
906 dp = map_domain_page(d_frame);
908 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
910 unmap_domain_page(dp);
911 unmap_domain_page(sp);
913 put_page_and_type(mfn_to_page(d_frame));
914 error_out:
915 if ( have_s_ref )
916 put_page(mfn_to_page(s_frame));
917 if ( have_s_grant )
918 __release_grant_for_copy(sd, op->source.u.ref, 1);
919 if ( have_d_grant )
920 __release_grant_for_copy(dd, op->dest.u.ref, 0);
921 if ( sd )
922 put_domain(sd);
923 if ( dd )
924 put_domain(dd);
925 op->status = rc;
926 }
928 static long
929 gnttab_copy(
930 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
931 {
932 int i;
933 struct gnttab_copy op;
935 for ( i = 0; i < count; i++ )
936 {
937 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
938 return -EFAULT;
939 __gnttab_copy(&op);
940 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
941 return -EFAULT;
942 }
943 return 0;
944 }
946 long
947 do_grant_table_op(
948 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
949 {
950 long rc;
951 struct domain *d = current->domain;
953 if ( count > 512 )
954 return -EINVAL;
956 LOCK_BIGLOCK(d);
958 rc = -EFAULT;
959 switch ( cmd )
960 {
961 case GNTTABOP_map_grant_ref:
962 {
963 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
964 guest_handle_cast(uop, gnttab_map_grant_ref_t);
965 if ( unlikely(!guest_handle_okay(map, count)) )
966 goto out;
967 rc = gnttab_map_grant_ref(map, count);
968 break;
969 }
970 case GNTTABOP_unmap_grant_ref:
971 {
972 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
973 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
974 if ( unlikely(!guest_handle_okay(unmap, count)) )
975 goto out;
976 rc = gnttab_unmap_grant_ref(unmap, count);
977 break;
978 }
979 case GNTTABOP_setup_table:
980 {
981 rc = gnttab_setup_table(
982 guest_handle_cast(uop, gnttab_setup_table_t), count);
983 break;
984 }
985 case GNTTABOP_transfer:
986 {
987 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
988 guest_handle_cast(uop, gnttab_transfer_t);
989 if ( unlikely(!guest_handle_okay(transfer, count)) )
990 goto out;
991 rc = gnttab_transfer(transfer, count);
992 break;
993 }
994 case GNTTABOP_copy:
995 {
996 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
997 guest_handle_cast(uop, gnttab_copy_t);
998 if ( unlikely(!guest_handle_okay(copy, count)) )
999 goto out;
1000 rc = gnttab_copy(copy, count);
1001 break;
1003 default:
1004 rc = -ENOSYS;
1005 break;
1008 out:
1009 UNLOCK_BIGLOCK(d);
1011 return rc;
1014 int
1015 grant_table_create(
1016 struct domain *d)
1018 struct grant_table *t;
1019 int i;
1021 BUG_ON(MAPTRACK_MAX_ENTRIES < NR_GRANT_ENTRIES);
1022 if ( (t = xmalloc(struct grant_table)) == NULL )
1023 goto no_mem;
1025 /* Simple stuff. */
1026 memset(t, 0, sizeof(*t));
1027 spin_lock_init(&t->lock);
1029 /* Active grant table. */
1030 t->active = xmalloc_array(struct active_grant_entry, NR_GRANT_ENTRIES);
1031 if ( t->active == NULL )
1032 goto no_mem;
1033 memset(t->active, 0, sizeof(struct active_grant_entry) * NR_GRANT_ENTRIES);
1035 /* Tracking of mapped foreign frames table */
1036 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
1037 goto no_mem;
1038 t->maptrack_order = 0;
1039 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
1040 memset(t->maptrack, 0, PAGE_SIZE);
1041 for ( i = 0; i < t->maptrack_limit; i++ )
1042 t->maptrack[i].ref = i+1;
1044 /* Shared grant table. */
1045 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1046 if ( t->shared == NULL )
1047 goto no_mem;
1048 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1050 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1051 gnttab_create_shared_page(d, t, i);
1053 /* Okay, install the structure. */
1054 wmb(); /* avoid races with lock-free access to d->grant_table */
1055 d->grant_table = t;
1056 return 0;
1058 no_mem:
1059 if ( t != NULL )
1061 xfree(t->active);
1062 free_xenheap_page(t->maptrack);
1063 xfree(t);
1065 return -ENOMEM;
1068 void
1069 gnttab_release_mappings(
1070 struct domain *d)
1072 struct grant_table *gt = d->grant_table;
1073 struct grant_mapping *map;
1074 grant_ref_t ref;
1075 grant_handle_t handle;
1076 struct domain *rd;
1077 struct active_grant_entry *act;
1078 struct grant_entry *sha;
1080 BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
1082 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1084 map = &gt->maptrack[handle];
1085 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
1086 continue;
1088 ref = map->ref;
1090 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1091 handle, ref, map->flags, map->domid);
1093 rd = find_domain_by_id(map->domid);
1094 BUG_ON(rd == NULL);
1096 spin_lock(&rd->grant_table->lock);
1098 act = &rd->grant_table->active[ref];
1099 sha = &rd->grant_table->shared[ref];
1101 if ( map->flags & GNTMAP_readonly )
1103 if ( map->flags & GNTMAP_device_map )
1105 BUG_ON(!(act->pin & GNTPIN_devr_mask));
1106 act->pin -= GNTPIN_devr_inc;
1107 put_page(mfn_to_page(act->frame));
1110 if ( map->flags & GNTMAP_host_map )
1112 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
1113 act->pin -= GNTPIN_hstr_inc;
1114 /* Done implicitly when page tables are destroyed. */
1115 /* put_page(mfn_to_page(act->frame)); */
1118 else
1120 if ( map->flags & GNTMAP_device_map )
1122 BUG_ON(!(act->pin & GNTPIN_devw_mask));
1123 act->pin -= GNTPIN_devw_inc;
1124 put_page_and_type(mfn_to_page(act->frame));
1127 if ( map->flags & GNTMAP_host_map )
1129 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
1130 act->pin -= GNTPIN_hstw_inc;
1131 /* Done implicitly when page tables are destroyed. */
1132 /* put_page_and_type(mfn_to_page(act->frame)); */
1135 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
1136 gnttab_clear_flag(_GTF_writing, &sha->flags);
1139 if ( act->pin == 0 )
1140 gnttab_clear_flag(_GTF_reading, &sha->flags);
1142 spin_unlock(&rd->grant_table->lock);
1144 put_domain(rd);
1146 map->flags = 0;
1151 void
1152 grant_table_destroy(
1153 struct domain *d)
1155 struct grant_table *t = d->grant_table;
1157 if ( t == NULL )
1158 return;
1160 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1161 free_xenheap_page(t->maptrack);
1162 xfree(t->active);
1163 xfree(t);
1165 d->grant_table = NULL;
1168 /*
1169 * Local variables:
1170 * mode: C
1171 * c-set-style: "BSD"
1172 * c-basic-offset: 4
1173 * tab-width: 4
1174 * indent-tabs-mode: nil
1175 * End:
1176 */