ia64/xen-unstable

view xen/common/grant_table.c @ 11128:f2f584093379

[POWERPC] Update .hgignore
Signed-off-by: Hollis Blanchard <hollisb@us.ibm.com>
author kfraser@localhost.localdomain
date Tue Aug 15 10:38:59 2006 +0100 (2006-08-15)
parents 1b85fbc8c013
children 0f917d63e960
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/lib.h>
28 #include <xen/sched.h>
29 #include <xen/shadow.h>
30 #include <xen/mm.h>
31 #include <xen/trace.h>
32 #include <xen/guest_access.h>
33 #include <xen/domain_page.h>
34 #include <acm/acm_hooks.h>
36 /*
37 * The first two members of a grant entry are updated as a combined pair.
38 * The following union allows that to happen in an endian-neutral fashion.
39 */
40 union grant_combo {
41 uint32_t word;
42 struct {
43 uint16_t flags;
44 domid_t domid;
45 } shorts;
46 };
48 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
49 do { \
50 DPRINTK( _f, ## _a ); \
51 rc = (_rc); \
52 goto _lbl; \
53 } while ( 0 )
55 static inline int
56 get_maptrack_handle(
57 struct grant_table *t)
58 {
59 unsigned int h;
60 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
61 return -1;
62 t->maptrack_head = t->maptrack[h].ref;
63 t->map_count++;
64 return h;
65 }
67 static inline void
68 put_maptrack_handle(
69 struct grant_table *t, int handle)
70 {
71 t->maptrack[handle].ref = t->maptrack_head;
72 t->maptrack_head = handle;
73 t->map_count--;
74 }
76 /*
77 * Returns 0 if TLB flush / invalidate required by caller.
78 * va will indicate the address to be invalidated.
79 *
80 * addr is _either_ a host virtual address, or the address of the pte to
81 * update, as indicated by the GNTMAP_contains_pte flag.
82 */
83 static void
84 __gnttab_map_grant_ref(
85 struct gnttab_map_grant_ref *op)
86 {
87 struct domain *ld, *rd;
88 struct vcpu *led;
89 int handle;
90 unsigned long frame = 0;
91 int rc = GNTST_okay;
92 struct active_grant_entry *act;
94 /* Entry details from @rd's shared grant table. */
95 grant_entry_t *sha;
96 domid_t sdom;
97 u16 sflags;
99 /*
100 * We bound the number of times we retry CMPXCHG on memory locations that
101 * we share with a guest OS. The reason is that the guest can modify that
102 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
103 * could cause us to livelock. There are a few cases where it is valid for
104 * the guest to race our updates (e.g., to change the GTF_readonly flag),
105 * so we allow a few retries before failing.
106 */
107 int retries = 0;
109 led = current;
110 ld = led->domain;
112 if ( unlikely(op->ref >= NR_GRANT_ENTRIES) ||
113 unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
114 {
115 DPRINTK("Bad ref (%d) or flags (%x).\n", op->ref, op->flags);
116 op->status = GNTST_bad_gntref;
117 return;
118 }
120 if ( acm_pre_grant_map_ref(op->dom) )
121 {
122 op->status = GNTST_permission_denied;
123 return;
124 }
126 if ( unlikely((rd = find_domain_by_id(op->dom)) == NULL) )
127 {
128 if ( rd != NULL )
129 put_domain(rd);
130 DPRINTK("Could not find domain %d\n", op->dom);
131 op->status = GNTST_bad_domain;
132 return;
133 }
135 /* Get a maptrack handle. */
136 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
137 {
138 int i;
139 struct grant_mapping *new_mt;
140 struct grant_table *lgt = ld->grant_table;
142 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
143 {
144 put_domain(rd);
145 DPRINTK("Maptrack table is at maximum size.\n");
146 op->status = GNTST_no_device_space;
147 return;
148 }
150 /* Grow the maptrack table. */
151 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
152 if ( new_mt == NULL )
153 {
154 put_domain(rd);
155 DPRINTK("No more map handles available.\n");
156 op->status = GNTST_no_device_space;
157 return;
158 }
160 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
161 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
162 new_mt[i].ref = i+1;
164 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
165 lgt->maptrack = new_mt;
166 lgt->maptrack_order += 1;
167 lgt->maptrack_limit <<= 1;
169 DPRINTK("Doubled maptrack size\n");
170 handle = get_maptrack_handle(ld->grant_table);
171 }
173 act = &rd->grant_table->active[op->ref];
174 sha = &rd->grant_table->shared[op->ref];
176 spin_lock(&rd->grant_table->lock);
178 if ( !act->pin ||
179 (!(op->flags & GNTMAP_readonly) &&
180 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
181 {
182 sflags = sha->flags;
183 sdom = sha->domid;
185 /*
186 * This loop attempts to set the access (reading/writing) flags
187 * in the grant table entry. It tries a cmpxchg on the field
188 * up to five times, and then fails under the assumption that
189 * the guest is misbehaving.
190 */
191 for ( ; ; )
192 {
193 union grant_combo scombo, prev_scombo, new_scombo;
195 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access) ||
196 unlikely(sdom != led->domain->domain_id) )
197 PIN_FAIL(unlock_out, GNTST_general_error,
198 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
199 sflags, sdom, led->domain->domain_id);
201 /* Merge two 16-bit values into a 32-bit combined update. */
202 scombo.shorts.flags = sflags;
203 scombo.shorts.domid = sdom;
205 new_scombo = scombo;
206 new_scombo.shorts.flags |= GTF_reading;
208 if ( !(op->flags & GNTMAP_readonly) )
209 {
210 new_scombo.shorts.flags |= GTF_writing;
211 if ( unlikely(sflags & GTF_readonly) )
212 PIN_FAIL(unlock_out, GNTST_general_error,
213 "Attempt to write-pin a r/o grant entry.\n");
214 }
216 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
217 scombo.word, new_scombo.word);
219 /* Did the combined update work (did we see what we expected?). */
220 if ( likely(prev_scombo.word == scombo.word) )
221 break;
223 if ( retries++ == 4 )
224 PIN_FAIL(unlock_out, GNTST_general_error,
225 "Shared grant entry is unstable.\n");
227 /* Didn't see what we expected. Split out the seen flags & dom. */
228 sflags = prev_scombo.shorts.flags;
229 sdom = prev_scombo.shorts.domid;
230 }
232 if ( !act->pin )
233 {
234 act->domid = sdom;
235 act->frame = gmfn_to_mfn(rd, sha->frame);
236 }
237 }
238 else if ( (act->pin & 0x80808080U) != 0 )
239 PIN_FAIL(unlock_out, ENOSPC,
240 "Risk of counter overflow %08x\n", act->pin);
242 if ( op->flags & GNTMAP_device_map )
243 act->pin += (op->flags & GNTMAP_readonly) ?
244 GNTPIN_devr_inc : GNTPIN_devw_inc;
245 if ( op->flags & GNTMAP_host_map )
246 act->pin += (op->flags & GNTMAP_readonly) ?
247 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
249 spin_unlock(&rd->grant_table->lock);
251 frame = act->frame;
252 if ( unlikely(!mfn_valid(frame)) ||
253 unlikely(!((op->flags & GNTMAP_readonly) ?
254 get_page(mfn_to_page(frame), rd) :
255 get_page_and_type(mfn_to_page(frame), rd,
256 PGT_writable_page))) )
257 PIN_FAIL(undo_out, GNTST_general_error,
258 "Could not pin the granted frame (%lx)!\n", frame);
260 if ( op->flags & GNTMAP_host_map )
261 {
262 rc = create_grant_host_mapping(op->host_addr, frame, op->flags);
263 if ( rc != GNTST_okay )
264 {
265 if ( !(op->flags & GNTMAP_readonly) )
266 put_page_type(mfn_to_page(frame));
267 put_page(mfn_to_page(frame));
268 goto undo_out;
269 }
271 if ( op->flags & GNTMAP_device_map )
272 {
273 (void)get_page(mfn_to_page(frame), rd);
274 if ( !(op->flags & GNTMAP_readonly) )
275 get_page_type(mfn_to_page(frame), PGT_writable_page);
276 }
277 }
279 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
281 ld->grant_table->maptrack[handle].domid = op->dom;
282 ld->grant_table->maptrack[handle].ref = op->ref;
283 ld->grant_table->maptrack[handle].flags = op->flags;
285 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
286 op->handle = handle;
287 op->status = GNTST_okay;
289 put_domain(rd);
290 return;
292 undo_out:
293 spin_lock(&rd->grant_table->lock);
295 if ( op->flags & GNTMAP_device_map )
296 act->pin -= (op->flags & GNTMAP_readonly) ?
297 GNTPIN_devr_inc : GNTPIN_devw_inc;
298 if ( op->flags & GNTMAP_host_map )
299 act->pin -= (op->flags & GNTMAP_readonly) ?
300 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
302 if ( !(op->flags & GNTMAP_readonly) &&
303 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
304 gnttab_clear_flag(_GTF_writing, &sha->flags);
306 if ( !act->pin )
307 gnttab_clear_flag(_GTF_reading, &sha->flags);
309 unlock_out:
310 spin_unlock(&rd->grant_table->lock);
311 op->status = rc;
312 put_maptrack_handle(ld->grant_table, handle);
313 put_domain(rd);
314 }
316 static long
317 gnttab_map_grant_ref(
318 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
319 {
320 int i;
321 struct gnttab_map_grant_ref op;
323 for ( i = 0; i < count; i++ )
324 {
325 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
326 return -EFAULT;
327 __gnttab_map_grant_ref(&op);
328 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
329 return -EFAULT;
330 }
332 return 0;
333 }
335 static void
336 __gnttab_unmap_grant_ref(
337 struct gnttab_unmap_grant_ref *op)
338 {
339 domid_t dom;
340 grant_ref_t ref;
341 struct domain *ld, *rd;
342 struct active_grant_entry *act;
343 grant_entry_t *sha;
344 struct grant_mapping *map;
345 u16 flags;
346 s16 rc = 0;
347 unsigned long frame;
349 ld = current->domain;
351 frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
353 map = &ld->grant_table->maptrack[op->handle];
355 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) ||
356 unlikely(!map->flags) )
357 {
358 DPRINTK("Bad handle (%d).\n", op->handle);
359 op->status = GNTST_bad_handle;
360 return;
361 }
363 dom = map->domid;
364 ref = map->ref;
365 flags = map->flags;
367 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) )
368 {
369 if ( rd != NULL )
370 put_domain(rd);
371 DPRINTK("Could not find domain %d\n", dom);
372 op->status = GNTST_bad_domain;
373 return;
374 }
376 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
378 act = &rd->grant_table->active[ref];
379 sha = &rd->grant_table->shared[ref];
381 spin_lock(&rd->grant_table->lock);
383 if ( frame == 0 )
384 {
385 frame = act->frame;
386 }
387 else
388 {
389 if ( unlikely(frame != act->frame) )
390 PIN_FAIL(unmap_out, GNTST_general_error,
391 "Bad frame number doesn't match gntref.\n");
392 if ( flags & GNTMAP_device_map )
393 {
394 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
395 map->flags &= ~GNTMAP_device_map;
396 if ( flags & GNTMAP_readonly )
397 {
398 act->pin -= GNTPIN_devr_inc;
399 put_page(mfn_to_page(frame));
400 }
401 else
402 {
403 act->pin -= GNTPIN_devw_inc;
404 put_page_and_type(mfn_to_page(frame));
405 }
406 }
407 }
409 if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
410 {
411 if ( (rc = destroy_grant_host_mapping(op->host_addr,
412 frame, flags)) < 0 )
413 goto unmap_out;
415 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
416 map->flags &= ~GNTMAP_host_map;
417 if ( flags & GNTMAP_readonly )
418 {
419 act->pin -= GNTPIN_hstr_inc;
420 put_page(mfn_to_page(frame));
421 }
422 else
423 {
424 act->pin -= GNTPIN_hstw_inc;
425 put_page_and_type(mfn_to_page(frame));
426 }
427 }
429 if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
430 {
431 map->flags = 0;
432 put_maptrack_handle(ld->grant_table, op->handle);
433 }
435 /* If just unmapped a writable mapping, mark as dirtied */
436 if ( !(flags & GNTMAP_readonly) )
437 gnttab_log_dirty(rd, frame);
439 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
440 !(flags & GNTMAP_readonly) )
441 gnttab_clear_flag(_GTF_writing, &sha->flags);
443 if ( act->pin == 0 )
444 gnttab_clear_flag(_GTF_reading, &sha->flags);
446 unmap_out:
447 op->status = rc;
448 spin_unlock(&rd->grant_table->lock);
449 put_domain(rd);
450 }
452 static long
453 gnttab_unmap_grant_ref(
454 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
455 {
456 int i;
457 struct gnttab_unmap_grant_ref op;
459 for ( i = 0; i < count; i++ )
460 {
461 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
462 goto fault;
463 __gnttab_unmap_grant_ref(&op);
464 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
465 goto fault;
466 }
468 flush_tlb_mask(current->domain->domain_dirty_cpumask);
469 return 0;
471 fault:
472 flush_tlb_mask(current->domain->domain_dirty_cpumask);
473 return -EFAULT;
474 }
476 static long
477 gnttab_setup_table(
478 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
479 {
480 struct gnttab_setup_table op;
481 struct domain *d;
482 int i;
483 unsigned long gmfn;
484 domid_t dom;
486 if ( count != 1 )
487 return -EINVAL;
489 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
490 {
491 DPRINTK("Fault while reading gnttab_setup_table_t.\n");
492 return -EFAULT;
493 }
495 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
496 {
497 DPRINTK("Xen only supports up to %d grant-table frames per domain.\n",
498 NR_GRANT_FRAMES);
499 op.status = GNTST_general_error;
500 goto out;
501 }
503 dom = op.dom;
504 if ( dom == DOMID_SELF )
505 {
506 dom = current->domain->domain_id;
507 }
508 else if ( unlikely(!IS_PRIV(current->domain)) )
509 {
510 op.status = GNTST_permission_denied;
511 goto out;
512 }
514 if ( unlikely((d = find_domain_by_id(dom)) == NULL) )
515 {
516 DPRINTK("Bad domid %d.\n", dom);
517 op.status = GNTST_bad_domain;
518 goto out;
519 }
521 ASSERT(d->grant_table != NULL);
522 op.status = GNTST_okay;
523 for ( i = 0; i < op.nr_frames; i++ )
524 {
525 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
526 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
527 }
529 put_domain(d);
531 out:
532 if ( unlikely(copy_to_guest(uop, &op, 1)) )
533 return -EFAULT;
535 return 0;
536 }
538 /*
539 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
540 * ownership of a page frame. If so, lock down the grant entry.
541 */
542 static int
543 gnttab_prepare_for_transfer(
544 struct domain *rd, struct domain *ld, grant_ref_t ref)
545 {
546 struct grant_table *rgt;
547 struct grant_entry *sha;
548 domid_t sdom;
549 u16 sflags;
550 union grant_combo scombo, prev_scombo, tmp_scombo;
551 int retries = 0;
553 if ( unlikely((rgt = rd->grant_table) == NULL) ||
554 unlikely(ref >= NR_GRANT_ENTRIES) )
555 {
556 DPRINTK("Dom %d has no g.t., or ref is bad (%d).\n",
557 rd->domain_id, ref);
558 return 0;
559 }
561 spin_lock(&rgt->lock);
563 sha = &rgt->shared[ref];
565 sflags = sha->flags;
566 sdom = sha->domid;
568 for ( ; ; )
569 {
570 if ( unlikely(sflags != GTF_accept_transfer) ||
571 unlikely(sdom != ld->domain_id) )
572 {
573 DPRINTK("Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
574 sflags, sdom, ld->domain_id);
575 goto fail;
576 }
578 /* Merge two 16-bit values into a 32-bit combined update. */
579 scombo.shorts.flags = sflags;
580 scombo.shorts.domid = sdom;
582 tmp_scombo = scombo;
583 tmp_scombo.shorts.flags |= GTF_transfer_committed;
584 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
585 scombo.word, tmp_scombo.word);
587 /* Did the combined update work (did we see what we expected?). */
588 if ( likely(prev_scombo.word == scombo.word) )
589 break;
591 if ( retries++ == 4 )
592 {
593 DPRINTK("Shared grant entry is unstable.\n");
594 goto fail;
595 }
597 /* Didn't see what we expected. Split out the seen flags & dom. */
598 sflags = prev_scombo.shorts.flags;
599 sdom = prev_scombo.shorts.domid;
600 }
602 spin_unlock(&rgt->lock);
603 return 1;
605 fail:
606 spin_unlock(&rgt->lock);
607 return 0;
608 }
610 static long
611 gnttab_transfer(
612 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
613 {
614 struct domain *d = current->domain;
615 struct domain *e;
616 struct page_info *page;
617 int i;
618 grant_entry_t *sha;
619 struct gnttab_transfer gop;
620 unsigned long mfn;
622 for ( i = 0; i < count; i++ )
623 {
624 /* Read from caller address space. */
625 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
626 {
627 DPRINTK("gnttab_transfer: error reading req %d/%d\n", i, count);
628 return -EFAULT;
629 }
631 mfn = gmfn_to_mfn(d, gop.mfn);
633 /* Check the passed page frame for basic validity. */
634 if ( unlikely(!mfn_valid(mfn)) )
635 {
636 DPRINTK("gnttab_transfer: out-of-range %lx\n",
637 (unsigned long)gop.mfn);
638 gop.status = GNTST_bad_page;
639 goto copyback;
640 }
642 page = mfn_to_page(mfn);
643 if ( unlikely(IS_XEN_HEAP_FRAME(page)) )
644 {
645 DPRINTK("gnttab_transfer: xen frame %lx\n",
646 (unsigned long)gop.mfn);
647 gop.status = GNTST_bad_page;
648 goto copyback;
649 }
651 if ( steal_page(d, page, 0) < 0 )
652 {
653 gop.status = GNTST_bad_page;
654 goto copyback;
655 }
657 /* Find the target domain. */
658 if ( unlikely((e = find_domain_by_id(gop.domid)) == NULL) )
659 {
660 DPRINTK("gnttab_transfer: can't find domain %d\n", gop.domid);
661 page->count_info &= ~(PGC_count_mask|PGC_allocated);
662 free_domheap_page(page);
663 gop.status = GNTST_bad_domain;
664 goto copyback;
665 }
667 spin_lock(&e->page_alloc_lock);
669 /*
670 * Check that 'e' will accept the page and has reservation
671 * headroom. Also, a domain mustn't have PGC_allocated
672 * pages when it is dying.
673 */
674 if ( unlikely(test_bit(_DOMF_dying, &e->domain_flags)) ||
675 unlikely(e->tot_pages >= e->max_pages) ||
676 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
677 {
678 if ( !test_bit(_DOMF_dying, &e->domain_flags) )
679 DPRINTK("gnttab_transfer: Transferee has no reservation "
680 "headroom (%d,%d) or provided a bad grant ref (%08x) "
681 "or is dying (%lx)\n",
682 e->tot_pages, e->max_pages, gop.ref, e->domain_flags);
683 spin_unlock(&e->page_alloc_lock);
684 put_domain(e);
685 page->count_info &= ~(PGC_count_mask|PGC_allocated);
686 free_domheap_page(page);
687 gop.status = GNTST_general_error;
688 goto copyback;
689 }
691 /* Okay, add the page to 'e'. */
692 if ( unlikely(e->tot_pages++ == 0) )
693 get_knownalive_domain(e);
694 list_add_tail(&page->list, &e->page_list);
695 page_set_owner(page, e);
697 spin_unlock(&e->page_alloc_lock);
699 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
701 /* Tell the guest about its new page frame. */
702 sha = &e->grant_table->shared[gop.ref];
703 guest_physmap_add_page(e, sha->frame, mfn);
704 sha->frame = mfn;
705 wmb();
706 sha->flags |= GTF_transfer_completed;
708 put_domain(e);
710 gop.status = GNTST_okay;
712 copyback:
713 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
714 {
715 DPRINTK("gnttab_transfer: error writing resp %d/%d\n", i, count);
716 return -EFAULT;
717 }
718 }
720 return 0;
721 }
723 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
724 type and reference counts. */
725 static void
726 __release_grant_for_copy(
727 struct domain *rd, unsigned long gref, int readonly)
728 {
729 grant_entry_t *const sha = &rd->grant_table->shared[gref];
730 struct active_grant_entry *const act = &rd->grant_table->active[gref];
731 const unsigned long r_frame = act->frame;
733 if ( !readonly )
734 gnttab_log_dirty(rd, r_frame);
736 spin_lock(&rd->grant_table->lock);
737 if ( readonly )
738 act->pin -= GNTPIN_hstr_inc;
739 else
740 act->pin -= GNTPIN_hstw_inc;
742 if ( !(act->pin & GNTPIN_hstw_mask) && !readonly )
743 gnttab_clear_flag(_GTF_writing, &sha->flags);
745 if ( !act->pin )
746 gnttab_clear_flag(_GTF_reading, &sha->flags);
747 spin_unlock(&rd->grant_table->lock);
748 }
750 /* Grab a frame number from a grant entry and update the flags and pin
751 count as appropriate. Note that this does *not* update the page
752 type or reference counts. */
753 static int
754 __acquire_grant_for_copy(
755 struct domain *rd, unsigned long gref, int readonly,
756 unsigned long *frame)
757 {
758 grant_entry_t *sha;
759 struct active_grant_entry *act;
760 s16 rc = GNTST_okay;
761 int retries = 0;
762 u16 sflags;
763 domid_t sdom;
765 if ( unlikely(gref >= NR_GRANT_ENTRIES) )
766 PIN_FAIL(error_out, GNTST_bad_gntref,
767 "Bad grant reference %ld\n", gref);
769 act = &rd->grant_table->active[gref];
770 sha = &rd->grant_table->shared[gref];
772 spin_lock(&rd->grant_table->lock);
774 if ( !act->pin ||
775 (!readonly && !(act->pin & GNTPIN_hstw_mask)) )
776 {
777 sflags = sha->flags;
778 sdom = sha->domid;
780 for ( ; ; )
781 {
782 union grant_combo scombo, prev_scombo, new_scombo;
784 if ( unlikely((sflags & GTF_type_mask) != GTF_permit_access ||
785 sdom != current->domain->domain_id ) )
786 PIN_FAIL(unlock_out, GNTST_general_error,
787 "Bad flags (%x) or dom (%d). (NB. expected dom %d)\n",
788 sflags, sdom, current->domain->domain_id);
790 /* Merge two 16-bit values into a 32-bit combined update. */
791 scombo.shorts.flags = sflags;
792 scombo.shorts.domid = sdom;
794 new_scombo = scombo;
795 new_scombo.shorts.flags |= GTF_reading;
797 if ( !readonly )
798 {
799 new_scombo.shorts.flags |= GTF_writing;
800 if ( unlikely(sflags & GTF_readonly) )
801 PIN_FAIL(unlock_out, GNTST_general_error,
802 "Attempt to write-pin a r/o grant entry.\n");
803 }
804 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
805 scombo.word, new_scombo.word);
806 if ( likely(prev_scombo.word == scombo.word) )
807 break;
809 if ( retries++ == 4 )
810 PIN_FAIL(unlock_out, GNTST_general_error,
811 "Shared grant entry is unstable.\n");
812 sflags = prev_scombo.shorts.flags;
813 sdom = prev_scombo.shorts.flags;
814 }
816 if ( !act->pin )
817 {
818 act->domid = sdom;
819 act->frame = gmfn_to_mfn(rd, sha->frame);
820 }
821 }
822 else if ( (act->pin & 0x80808080U) != 0 )
823 PIN_FAIL(unlock_out, ENOSPC,
824 "Risk of counter overflow %08x\n", act->pin);
826 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
828 *frame = act->frame;
830 unlock_out:
831 spin_unlock(&rd->grant_table->lock);
832 error_out:
833 return rc;
834 }
836 static void
837 __gnttab_copy(
838 struct gnttab_copy *op)
839 {
840 struct domain *sd = NULL, *dd = NULL;
841 unsigned long s_frame, d_frame;
842 char *sp, *dp;
843 s16 rc = GNTST_okay;
844 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
845 int src_is_gref, dest_is_gref;
847 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
848 ((op->dest.offset + op->len) > PAGE_SIZE) )
849 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
851 src_is_gref = op->flags & GNTCOPY_source_gref;
852 dest_is_gref = op->flags & GNTCOPY_dest_gref;
854 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
855 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
856 PIN_FAIL(error_out, GNTST_permission_denied,
857 "only allow copy-by-mfn for DOMID_SELF.\n");
859 if ( op->source.domid == DOMID_SELF )
860 {
861 sd = current->domain;
862 get_knownalive_domain(sd);
863 }
864 else if ( (sd = find_domain_by_id(op->source.domid)) == NULL )
865 {
866 PIN_FAIL(error_out, GNTST_bad_domain,
867 "couldn't find %d\n", op->source.domid);
868 }
870 if ( op->dest.domid == DOMID_SELF )
871 {
872 dd = current->domain;
873 get_knownalive_domain(dd);
874 }
875 else if ( (dd = find_domain_by_id(op->dest.domid)) == NULL )
876 {
877 PIN_FAIL(error_out, GNTST_bad_domain,
878 "couldn't find %d\n", op->dest.domid);
879 }
881 if ( src_is_gref )
882 {
883 rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame);
884 if ( rc != GNTST_okay )
885 goto error_out;
886 have_s_grant = 1;
887 }
888 else
889 {
890 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
891 }
892 if ( !get_page(mfn_to_page(s_frame), sd) )
893 PIN_FAIL(error_out, GNTST_general_error,
894 "could not get source frame %lx.\n", s_frame);
895 have_s_ref = 1;
897 if ( dest_is_gref )
898 {
899 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame);
900 if ( rc != GNTST_okay )
901 goto error_out;
902 have_d_grant = 1;
903 }
904 else
905 {
906 d_frame = gmfn_to_mfn(sd, op->dest.u.gmfn);
907 }
908 if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) )
909 PIN_FAIL(error_out, GNTST_general_error,
910 "could not get source frame %lx.\n", d_frame);
912 sp = map_domain_page(s_frame);
913 dp = map_domain_page(d_frame);
915 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
917 unmap_domain_page(dp);
918 unmap_domain_page(sp);
920 put_page_and_type(mfn_to_page(d_frame));
921 error_out:
922 if ( have_s_ref )
923 put_page(mfn_to_page(s_frame));
924 if ( have_s_grant )
925 __release_grant_for_copy(sd, op->source.u.ref, 1);
926 if ( have_d_grant )
927 __release_grant_for_copy(dd, op->dest.u.ref, 0);
928 if ( sd )
929 put_domain(sd);
930 if ( dd )
931 put_domain(dd);
932 op->status = rc;
933 }
935 static long
936 gnttab_copy(
937 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
938 {
939 int i;
940 struct gnttab_copy op;
942 for ( i = 0; i < count; i++ )
943 {
944 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
945 return -EFAULT;
946 __gnttab_copy(&op);
947 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
948 return -EFAULT;
949 }
950 return 0;
951 }
953 long
954 do_grant_table_op(
955 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
956 {
957 long rc;
958 struct domain *d = current->domain;
960 if ( count > 512 )
961 return -EINVAL;
963 LOCK_BIGLOCK(d);
965 rc = -EFAULT;
966 switch ( cmd )
967 {
968 case GNTTABOP_map_grant_ref:
969 {
970 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
971 guest_handle_cast(uop, gnttab_map_grant_ref_t);
972 if ( unlikely(!guest_handle_okay(map, count)) )
973 goto out;
974 rc = gnttab_map_grant_ref(map, count);
975 break;
976 }
977 case GNTTABOP_unmap_grant_ref:
978 {
979 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
980 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
981 if ( unlikely(!guest_handle_okay(unmap, count)) )
982 goto out;
983 rc = gnttab_unmap_grant_ref(unmap, count);
984 break;
985 }
986 case GNTTABOP_setup_table:
987 {
988 rc = gnttab_setup_table(
989 guest_handle_cast(uop, gnttab_setup_table_t), count);
990 break;
991 }
992 case GNTTABOP_transfer:
993 {
994 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
995 guest_handle_cast(uop, gnttab_transfer_t);
996 if ( unlikely(!guest_handle_okay(transfer, count)) )
997 goto out;
998 rc = gnttab_transfer(transfer, count);
999 break;
1001 case GNTTABOP_copy:
1003 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
1004 guest_handle_cast(uop, gnttab_copy_t);
1005 if ( unlikely(!guest_handle_okay(copy, count)) )
1006 goto out;
1007 rc = gnttab_copy(copy, count);
1008 break;
1010 default:
1011 rc = -ENOSYS;
1012 break;
1015 out:
1016 UNLOCK_BIGLOCK(d);
1018 return rc;
1021 int
1022 grant_table_create(
1023 struct domain *d)
1025 struct grant_table *t;
1026 int i;
1028 BUG_ON(MAPTRACK_MAX_ENTRIES < NR_GRANT_ENTRIES);
1029 if ( (t = xmalloc(struct grant_table)) == NULL )
1030 goto no_mem;
1032 /* Simple stuff. */
1033 memset(t, 0, sizeof(*t));
1034 spin_lock_init(&t->lock);
1036 /* Active grant table. */
1037 t->active = xmalloc_array(struct active_grant_entry, NR_GRANT_ENTRIES);
1038 if ( t->active == NULL )
1039 goto no_mem;
1040 memset(t->active, 0, sizeof(struct active_grant_entry) * NR_GRANT_ENTRIES);
1042 /* Tracking of mapped foreign frames table */
1043 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
1044 goto no_mem;
1045 t->maptrack_order = 0;
1046 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
1047 memset(t->maptrack, 0, PAGE_SIZE);
1048 for ( i = 0; i < t->maptrack_limit; i++ )
1049 t->maptrack[i].ref = i+1;
1051 /* Shared grant table. */
1052 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1053 if ( t->shared == NULL )
1054 goto no_mem;
1055 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1057 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1058 gnttab_create_shared_page(d, t, i);
1060 /* Okay, install the structure. */
1061 wmb(); /* avoid races with lock-free access to d->grant_table */
1062 d->grant_table = t;
1063 return 0;
1065 no_mem:
1066 if ( t != NULL )
1068 xfree(t->active);
1069 free_xenheap_page(t->maptrack);
1070 xfree(t);
1072 return -ENOMEM;
1075 void
1076 gnttab_release_mappings(
1077 struct domain *d)
1079 struct grant_table *gt = d->grant_table;
1080 struct grant_mapping *map;
1081 grant_ref_t ref;
1082 grant_handle_t handle;
1083 struct domain *rd;
1084 struct active_grant_entry *act;
1085 struct grant_entry *sha;
1087 BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
1089 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1091 map = &gt->maptrack[handle];
1092 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
1093 continue;
1095 ref = map->ref;
1097 DPRINTK("Grant release (%hu) ref:(%hu) flags:(%x) dom:(%hu)\n",
1098 handle, ref, map->flags, map->domid);
1100 rd = find_domain_by_id(map->domid);
1101 BUG_ON(rd == NULL);
1103 spin_lock(&rd->grant_table->lock);
1105 act = &rd->grant_table->active[ref];
1106 sha = &rd->grant_table->shared[ref];
1108 if ( map->flags & GNTMAP_readonly )
1110 if ( map->flags & GNTMAP_device_map )
1112 BUG_ON(!(act->pin & GNTPIN_devr_mask));
1113 act->pin -= GNTPIN_devr_inc;
1114 put_page(mfn_to_page(act->frame));
1117 if ( map->flags & GNTMAP_host_map )
1119 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
1120 act->pin -= GNTPIN_hstr_inc;
1121 /* Done implicitly when page tables are destroyed. */
1122 /* put_page(mfn_to_page(act->frame)); */
1125 else
1127 if ( map->flags & GNTMAP_device_map )
1129 BUG_ON(!(act->pin & GNTPIN_devw_mask));
1130 act->pin -= GNTPIN_devw_inc;
1131 put_page_and_type(mfn_to_page(act->frame));
1134 if ( map->flags & GNTMAP_host_map )
1136 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
1137 act->pin -= GNTPIN_hstw_inc;
1138 /* Done implicitly when page tables are destroyed. */
1139 /* put_page_and_type(mfn_to_page(act->frame)); */
1142 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
1143 gnttab_clear_flag(_GTF_writing, &sha->flags);
1146 if ( act->pin == 0 )
1147 gnttab_clear_flag(_GTF_reading, &sha->flags);
1149 spin_unlock(&rd->grant_table->lock);
1151 put_domain(rd);
1153 map->flags = 0;
1158 void
1159 grant_table_destroy(
1160 struct domain *d)
1162 struct grant_table *t = d->grant_table;
1164 if ( t == NULL )
1165 return;
1167 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1168 free_xenheap_page(t->maptrack);
1169 xfree(t->active);
1170 xfree(t);
1172 d->grant_table = NULL;
1175 /*
1176 * Local variables:
1177 * mode: C
1178 * c-set-style: "BSD"
1179 * c-basic-offset: 4
1180 * tab-width: 4
1181 * indent-tabs-mode: nil
1182 * End:
1183 */