direct-io.hg

view xen/common/grant_table.c @ 12502:08007423788a

[XEN] Fix potential crash in grant-table code if
guest misbehaves and implicitly unmaps a PTE.
Signed-off-by: Keir Fraser <keir@xensource.com>
author kfraser@localhost.localdomain
date Tue Nov 21 16:14:53 2006 +0000 (2006-11-21)
parents 726960294d4b
children e1971b229e89
line source
1 /******************************************************************************
2 * common/grant_table.c
3 *
4 * Mechanism for granting foreign access to page frames, and receiving
5 * page-ownership transfers.
6 *
7 * Copyright (c) 2005 Christopher Clark
8 * Copyright (c) 2004 K A Fraser
9 * Copyright (c) 2005 Andrew Warfield
10 * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21 *
22 * You should have received a copy of the GNU General Public License
23 * along with this program; if not, write to the Free Software
24 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
25 */
27 #include <xen/config.h>
28 #include <xen/iocap.h>
29 #include <xen/lib.h>
30 #include <xen/sched.h>
31 #include <xen/shadow.h>
32 #include <xen/mm.h>
33 #include <xen/trace.h>
34 #include <xen/guest_access.h>
35 #include <xen/domain_page.h>
36 #include <acm/acm_hooks.h>
38 /*
39 * The first two members of a grant entry are updated as a combined pair.
40 * The following union allows that to happen in an endian-neutral fashion.
41 */
42 union grant_combo {
43 uint32_t word;
44 struct {
45 uint16_t flags;
46 domid_t domid;
47 } shorts;
48 };
50 #define PIN_FAIL(_lbl, _rc, _f, _a...) \
51 do { \
52 gdprintk(XENLOG_WARNING, _f, ## _a ); \
53 rc = (_rc); \
54 goto _lbl; \
55 } while ( 0 )
57 static inline int
58 get_maptrack_handle(
59 struct grant_table *t)
60 {
61 unsigned int h;
62 if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) )
63 return -1;
64 t->maptrack_head = t->maptrack[h].ref;
65 t->map_count++;
66 return h;
67 }
69 static inline void
70 put_maptrack_handle(
71 struct grant_table *t, int handle)
72 {
73 t->maptrack[handle].ref = t->maptrack_head;
74 t->maptrack_head = handle;
75 t->map_count--;
76 }
78 /*
79 * Returns 0 if TLB flush / invalidate required by caller.
80 * va will indicate the address to be invalidated.
81 *
82 * addr is _either_ a host virtual address, or the address of the pte to
83 * update, as indicated by the GNTMAP_contains_pte flag.
84 */
85 static void
86 __gnttab_map_grant_ref(
87 struct gnttab_map_grant_ref *op)
88 {
89 struct domain *ld, *rd;
90 struct vcpu *led;
91 int handle;
92 unsigned long frame = 0;
93 int rc = GNTST_okay;
94 struct active_grant_entry *act;
95 grant_entry_t *sha;
96 union grant_combo scombo, prev_scombo, new_scombo;
98 /*
99 * We bound the number of times we retry CMPXCHG on memory locations that
100 * we share with a guest OS. The reason is that the guest can modify that
101 * location at a higher rate than we can read-modify-CMPXCHG, so the guest
102 * could cause us to livelock. There are a few cases where it is valid for
103 * the guest to race our updates (e.g., to change the GTF_readonly flag),
104 * so we allow a few retries before failing.
105 */
106 int retries = 0;
108 led = current;
109 ld = led->domain;
111 if ( unlikely(op->ref >= NR_GRANT_ENTRIES) ||
112 unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) )
113 {
114 gdprintk(XENLOG_INFO, "Bad ref (%d) or flags (%x).\n",
115 op->ref, op->flags);
116 op->status = GNTST_bad_gntref;
117 return;
118 }
120 if ( acm_pre_grant_map_ref(op->dom) )
121 {
122 op->status = GNTST_permission_denied;
123 return;
124 }
126 if ( unlikely((rd = find_domain_by_id(op->dom)) == NULL) )
127 {
128 if ( rd != NULL )
129 put_domain(rd);
130 gdprintk(XENLOG_INFO, "Could not find domain %d\n", op->dom);
131 op->status = GNTST_bad_domain;
132 return;
133 }
135 /* Get a maptrack handle. */
136 if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) )
137 {
138 int i;
139 struct grant_mapping *new_mt;
140 struct grant_table *lgt = ld->grant_table;
142 if ( (lgt->maptrack_limit << 1) > MAPTRACK_MAX_ENTRIES )
143 {
144 put_domain(rd);
145 gdprintk(XENLOG_INFO, "Maptrack table is at maximum size.\n");
146 op->status = GNTST_no_device_space;
147 return;
148 }
150 /* Grow the maptrack table. */
151 new_mt = alloc_xenheap_pages(lgt->maptrack_order + 1);
152 if ( new_mt == NULL )
153 {
154 put_domain(rd);
155 gdprintk(XENLOG_INFO, "No more map handles available.\n");
156 op->status = GNTST_no_device_space;
157 return;
158 }
160 memcpy(new_mt, lgt->maptrack, PAGE_SIZE << lgt->maptrack_order);
161 for ( i = lgt->maptrack_limit; i < (lgt->maptrack_limit << 1); i++ )
162 {
163 new_mt[i].ref = i+1;
164 new_mt[i].flags = 0;
165 }
167 free_xenheap_pages(lgt->maptrack, lgt->maptrack_order);
168 lgt->maptrack = new_mt;
169 lgt->maptrack_order += 1;
170 lgt->maptrack_limit <<= 1;
172 gdprintk(XENLOG_INFO, "Doubled maptrack size\n");
173 handle = get_maptrack_handle(ld->grant_table);
174 }
176 act = &rd->grant_table->active[op->ref];
177 sha = &rd->grant_table->shared[op->ref];
179 spin_lock(&rd->grant_table->lock);
181 /* If already pinned, check the active domid and avoid refcnt overflow. */
182 if ( act->pin &&
183 ((act->domid != ld->domain_id) ||
184 (act->pin & 0x80808080U) != 0) )
185 PIN_FAIL(unlock_out, GNTST_general_error,
186 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
187 act->domid, ld->domain_id, act->pin);
189 if ( !act->pin ||
190 (!(op->flags & GNTMAP_readonly) &&
191 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) )
192 {
193 scombo.word = *(u32 *)&sha->flags;
195 /*
196 * This loop attempts to set the access (reading/writing) flags
197 * in the grant table entry. It tries a cmpxchg on the field
198 * up to five times, and then fails under the assumption that
199 * the guest is misbehaving.
200 */
201 for ( ; ; )
202 {
203 /* If not already pinned, check the grant domid and type. */
204 if ( !act->pin &&
205 (((scombo.shorts.flags & GTF_type_mask) !=
206 GTF_permit_access) ||
207 (scombo.shorts.domid != ld->domain_id)) )
208 PIN_FAIL(unlock_out, GNTST_general_error,
209 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
210 scombo.shorts.flags, scombo.shorts.domid,
211 ld->domain_id);
213 new_scombo = scombo;
214 new_scombo.shorts.flags |= GTF_reading;
216 if ( !(op->flags & GNTMAP_readonly) )
217 {
218 new_scombo.shorts.flags |= GTF_writing;
219 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
220 PIN_FAIL(unlock_out, GNTST_general_error,
221 "Attempt to write-pin a r/o grant entry.\n");
222 }
224 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
225 scombo.word, new_scombo.word);
226 if ( likely(prev_scombo.word == scombo.word) )
227 break;
229 if ( retries++ == 4 )
230 PIN_FAIL(unlock_out, GNTST_general_error,
231 "Shared grant entry is unstable.\n");
233 scombo = prev_scombo;
234 }
236 if ( !act->pin )
237 {
238 act->domid = scombo.shorts.domid;
239 act->frame = gmfn_to_mfn(rd, sha->frame);
240 }
241 }
243 if ( op->flags & GNTMAP_device_map )
244 act->pin += (op->flags & GNTMAP_readonly) ?
245 GNTPIN_devr_inc : GNTPIN_devw_inc;
246 if ( op->flags & GNTMAP_host_map )
247 act->pin += (op->flags & GNTMAP_readonly) ?
248 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
250 spin_unlock(&rd->grant_table->lock);
252 frame = act->frame;
253 if ( unlikely(!mfn_valid(frame)) ||
254 unlikely(!((op->flags & GNTMAP_readonly) ?
255 get_page(mfn_to_page(frame), rd) :
256 get_page_and_type(mfn_to_page(frame), rd,
257 PGT_writable_page))) )
258 {
259 if ( !test_bit(_DOMF_dying, &rd->domain_flags) )
260 gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n", frame);
261 rc = GNTST_general_error;
262 goto undo_out;
263 }
265 if ( op->flags & GNTMAP_host_map )
266 {
267 rc = create_grant_host_mapping(op->host_addr, frame, op->flags);
268 if ( rc != GNTST_okay )
269 {
270 if ( !(op->flags & GNTMAP_readonly) )
271 put_page_type(mfn_to_page(frame));
272 put_page(mfn_to_page(frame));
273 goto undo_out;
274 }
276 if ( op->flags & GNTMAP_device_map )
277 {
278 (void)get_page(mfn_to_page(frame), rd);
279 if ( !(op->flags & GNTMAP_readonly) )
280 get_page_type(mfn_to_page(frame), PGT_writable_page);
281 }
282 }
284 TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom);
286 ld->grant_table->maptrack[handle].domid = op->dom;
287 ld->grant_table->maptrack[handle].ref = op->ref;
288 ld->grant_table->maptrack[handle].flags = op->flags;
290 op->dev_bus_addr = (u64)frame << PAGE_SHIFT;
291 op->handle = handle;
292 op->status = GNTST_okay;
294 put_domain(rd);
295 return;
297 undo_out:
298 spin_lock(&rd->grant_table->lock);
300 if ( op->flags & GNTMAP_device_map )
301 act->pin -= (op->flags & GNTMAP_readonly) ?
302 GNTPIN_devr_inc : GNTPIN_devw_inc;
303 if ( op->flags & GNTMAP_host_map )
304 act->pin -= (op->flags & GNTMAP_readonly) ?
305 GNTPIN_hstr_inc : GNTPIN_hstw_inc;
307 if ( !(op->flags & GNTMAP_readonly) &&
308 !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) )
309 gnttab_clear_flag(_GTF_writing, &sha->flags);
311 if ( !act->pin )
312 gnttab_clear_flag(_GTF_reading, &sha->flags);
314 unlock_out:
315 spin_unlock(&rd->grant_table->lock);
316 op->status = rc;
317 put_maptrack_handle(ld->grant_table, handle);
318 put_domain(rd);
319 }
321 static long
322 gnttab_map_grant_ref(
323 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count)
324 {
325 int i;
326 struct gnttab_map_grant_ref op;
328 for ( i = 0; i < count; i++ )
329 {
330 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
331 return -EFAULT;
332 __gnttab_map_grant_ref(&op);
333 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
334 return -EFAULT;
335 }
337 return 0;
338 }
340 static void
341 __gnttab_unmap_grant_ref(
342 struct gnttab_unmap_grant_ref *op)
343 {
344 domid_t dom;
345 grant_ref_t ref;
346 struct domain *ld, *rd;
347 struct active_grant_entry *act;
348 grant_entry_t *sha;
349 struct grant_mapping *map;
350 u16 flags;
351 s16 rc = 0;
352 unsigned long frame;
354 ld = current->domain;
356 frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT);
358 map = &ld->grant_table->maptrack[op->handle];
360 if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) ||
361 unlikely(!map->flags) )
362 {
363 gdprintk(XENLOG_INFO, "Bad handle (%d).\n", op->handle);
364 op->status = GNTST_bad_handle;
365 return;
366 }
368 dom = map->domid;
369 ref = map->ref;
370 flags = map->flags;
372 if ( unlikely((rd = find_domain_by_id(dom)) == NULL) )
373 {
374 /* This can happen when a grant is implicitly unmapped. */
375 gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom);
376 domain_crash(ld); /* naughty... */
377 return;
378 }
380 TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom);
382 act = &rd->grant_table->active[ref];
383 sha = &rd->grant_table->shared[ref];
385 spin_lock(&rd->grant_table->lock);
387 if ( frame == 0 )
388 {
389 frame = act->frame;
390 }
391 else
392 {
393 if ( unlikely(frame != act->frame) )
394 PIN_FAIL(unmap_out, GNTST_general_error,
395 "Bad frame number doesn't match gntref.\n");
396 if ( flags & GNTMAP_device_map )
397 {
398 ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask));
399 map->flags &= ~GNTMAP_device_map;
400 if ( flags & GNTMAP_readonly )
401 {
402 act->pin -= GNTPIN_devr_inc;
403 put_page(mfn_to_page(frame));
404 }
405 else
406 {
407 act->pin -= GNTPIN_devw_inc;
408 put_page_and_type(mfn_to_page(frame));
409 }
410 }
411 }
413 if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
414 {
415 if ( (rc = destroy_grant_host_mapping(op->host_addr,
416 frame, flags)) < 0 )
417 goto unmap_out;
419 ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask));
420 map->flags &= ~GNTMAP_host_map;
421 if ( flags & GNTMAP_readonly )
422 {
423 act->pin -= GNTPIN_hstr_inc;
424 put_page(mfn_to_page(frame));
425 }
426 else
427 {
428 act->pin -= GNTPIN_hstw_inc;
429 put_page_and_type(mfn_to_page(frame));
430 }
431 }
433 if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 )
434 {
435 map->flags = 0;
436 put_maptrack_handle(ld->grant_table, op->handle);
437 }
439 /* If just unmapped a writable mapping, mark as dirtied */
440 if ( !(flags & GNTMAP_readonly) )
441 gnttab_mark_dirty(rd, frame);
443 if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) &&
444 !(flags & GNTMAP_readonly) )
445 gnttab_clear_flag(_GTF_writing, &sha->flags);
447 if ( act->pin == 0 )
448 gnttab_clear_flag(_GTF_reading, &sha->flags);
450 unmap_out:
451 op->status = rc;
452 spin_unlock(&rd->grant_table->lock);
453 put_domain(rd);
454 }
456 static long
457 gnttab_unmap_grant_ref(
458 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count)
459 {
460 int i;
461 struct gnttab_unmap_grant_ref op;
463 for ( i = 0; i < count; i++ )
464 {
465 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
466 goto fault;
467 __gnttab_unmap_grant_ref(&op);
468 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
469 goto fault;
470 }
472 flush_tlb_mask(current->domain->domain_dirty_cpumask);
473 return 0;
475 fault:
476 flush_tlb_mask(current->domain->domain_dirty_cpumask);
477 return -EFAULT;
478 }
480 static long
481 gnttab_setup_table(
482 XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count)
483 {
484 struct gnttab_setup_table op;
485 struct domain *d;
486 int i;
487 unsigned long gmfn;
488 domid_t dom;
490 if ( count != 1 )
491 return -EINVAL;
493 if ( unlikely(copy_from_guest(&op, uop, 1) != 0) )
494 {
495 gdprintk(XENLOG_INFO, "Fault while reading gnttab_setup_table_t.\n");
496 return -EFAULT;
497 }
499 if ( unlikely(op.nr_frames > NR_GRANT_FRAMES) )
500 {
501 gdprintk(XENLOG_INFO, "Xen only supports up to %d grant-table frames"
502 " per domain.\n",
503 NR_GRANT_FRAMES);
504 op.status = GNTST_general_error;
505 goto out;
506 }
508 dom = op.dom;
509 if ( dom == DOMID_SELF )
510 {
511 dom = current->domain->domain_id;
512 }
513 else if ( unlikely(!IS_PRIV(current->domain)) )
514 {
515 op.status = GNTST_permission_denied;
516 goto out;
517 }
519 if ( unlikely((d = find_domain_by_id(dom)) == NULL) )
520 {
521 gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom);
522 op.status = GNTST_bad_domain;
523 goto out;
524 }
526 ASSERT(d->grant_table != NULL);
527 op.status = GNTST_okay;
528 for ( i = 0; i < op.nr_frames; i++ )
529 {
530 gmfn = gnttab_shared_gmfn(d, d->grant_table, i);
531 (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1);
532 }
534 put_domain(d);
536 out:
537 if ( unlikely(copy_to_guest(uop, &op, 1)) )
538 return -EFAULT;
540 return 0;
541 }
543 /*
544 * Check that the given grant reference (rd,ref) allows 'ld' to transfer
545 * ownership of a page frame. If so, lock down the grant entry.
546 */
547 static int
548 gnttab_prepare_for_transfer(
549 struct domain *rd, struct domain *ld, grant_ref_t ref)
550 {
551 struct grant_table *rgt;
552 struct grant_entry *sha;
553 union grant_combo scombo, prev_scombo, new_scombo;
554 int retries = 0;
556 if ( unlikely((rgt = rd->grant_table) == NULL) ||
557 unlikely(ref >= NR_GRANT_ENTRIES) )
558 {
559 gdprintk(XENLOG_INFO, "Dom %d has no g.t., or ref is bad (%d).\n",
560 rd->domain_id, ref);
561 return 0;
562 }
564 spin_lock(&rgt->lock);
566 sha = &rgt->shared[ref];
568 scombo.word = *(u32 *)&sha->flags;
570 for ( ; ; )
571 {
572 if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) ||
573 unlikely(scombo.shorts.domid != ld->domain_id) )
574 {
575 gdprintk(XENLOG_INFO, "Bad flags (%x) or dom (%d). "
576 "(NB. expected dom %d)\n",
577 scombo.shorts.flags, scombo.shorts.domid,
578 ld->domain_id);
579 goto fail;
580 }
582 new_scombo = scombo;
583 new_scombo.shorts.flags |= GTF_transfer_committed;
585 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
586 scombo.word, new_scombo.word);
587 if ( likely(prev_scombo.word == scombo.word) )
588 break;
590 if ( retries++ == 4 )
591 {
592 gdprintk(XENLOG_WARNING, "Shared grant entry is unstable.\n");
593 goto fail;
594 }
596 scombo = prev_scombo;
597 }
599 spin_unlock(&rgt->lock);
600 return 1;
602 fail:
603 spin_unlock(&rgt->lock);
604 return 0;
605 }
607 static long
608 gnttab_transfer(
609 XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count)
610 {
611 struct domain *d = current->domain;
612 struct domain *e;
613 struct page_info *page;
614 int i;
615 grant_entry_t *sha;
616 struct gnttab_transfer gop;
617 unsigned long mfn;
619 for ( i = 0; i < count; i++ )
620 {
621 /* Read from caller address space. */
622 if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) )
623 {
624 gdprintk(XENLOG_INFO, "gnttab_transfer: error reading req %d/%d\n",
625 i, count);
626 return -EFAULT;
627 }
629 mfn = gmfn_to_mfn(d, gop.mfn);
631 /* Check the passed page frame for basic validity. */
632 if ( unlikely(!mfn_valid(mfn)) )
633 {
634 gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n",
635 (unsigned long)gop.mfn);
636 gop.status = GNTST_bad_page;
637 goto copyback;
638 }
640 page = mfn_to_page(mfn);
641 if ( unlikely(IS_XEN_HEAP_FRAME(page)) )
642 {
643 gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n",
644 (unsigned long)gop.mfn);
645 gop.status = GNTST_bad_page;
646 goto copyback;
647 }
649 if ( steal_page(d, page, 0) < 0 )
650 {
651 gop.status = GNTST_bad_page;
652 goto copyback;
653 }
655 /* Find the target domain. */
656 if ( unlikely((e = find_domain_by_id(gop.domid)) == NULL) )
657 {
658 gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n",
659 gop.domid);
660 page->count_info &= ~(PGC_count_mask|PGC_allocated);
661 free_domheap_page(page);
662 gop.status = GNTST_bad_domain;
663 goto copyback;
664 }
666 spin_lock(&e->page_alloc_lock);
668 /*
669 * Check that 'e' will accept the page and has reservation
670 * headroom. Also, a domain mustn't have PGC_allocated
671 * pages when it is dying.
672 */
673 if ( unlikely(test_bit(_DOMF_dying, &e->domain_flags)) ||
674 unlikely(e->tot_pages >= e->max_pages) ||
675 unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) )
676 {
677 if ( !test_bit(_DOMF_dying, &e->domain_flags) )
678 gdprintk(XENLOG_INFO, "gnttab_transfer: "
679 "Transferee has no reservation "
680 "headroom (%d,%d) or provided a bad grant ref (%08x) "
681 "or is dying (%lx)\n",
682 e->tot_pages, e->max_pages, gop.ref, e->domain_flags);
683 spin_unlock(&e->page_alloc_lock);
684 put_domain(e);
685 page->count_info &= ~(PGC_count_mask|PGC_allocated);
686 free_domheap_page(page);
687 gop.status = GNTST_general_error;
688 goto copyback;
689 }
691 /* Okay, add the page to 'e'. */
692 if ( unlikely(e->tot_pages++ == 0) )
693 get_knownalive_domain(e);
694 list_add_tail(&page->list, &e->page_list);
695 page_set_owner(page, e);
697 spin_unlock(&e->page_alloc_lock);
699 TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id);
701 /* Tell the guest about its new page frame. */
702 sha = &e->grant_table->shared[gop.ref];
703 guest_physmap_add_page(e, sha->frame, mfn);
704 sha->frame = mfn;
705 wmb();
706 sha->flags |= GTF_transfer_completed;
708 put_domain(e);
710 gop.status = GNTST_okay;
712 copyback:
713 if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) )
714 {
715 gdprintk(XENLOG_INFO, "gnttab_transfer: error writing resp %d/%d\n",
716 i, count);
717 return -EFAULT;
718 }
719 }
721 return 0;
722 }
724 /* Undo __acquire_grant_for_copy. Again, this has no effect on page
725 type and reference counts. */
726 static void
727 __release_grant_for_copy(
728 struct domain *rd, unsigned long gref, int readonly)
729 {
730 grant_entry_t *const sha = &rd->grant_table->shared[gref];
731 struct active_grant_entry *const act = &rd->grant_table->active[gref];
733 spin_lock(&rd->grant_table->lock);
735 if ( readonly )
736 {
737 act->pin -= GNTPIN_hstr_inc;
738 }
739 else
740 {
741 act->pin -= GNTPIN_hstw_inc;
742 if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) )
743 gnttab_clear_flag(_GTF_writing, &sha->flags);
744 }
746 if ( !act->pin )
747 gnttab_clear_flag(_GTF_reading, &sha->flags);
749 spin_unlock(&rd->grant_table->lock);
750 }
752 /* Grab a frame number from a grant entry and update the flags and pin
753 count as appropriate. Note that this does *not* update the page
754 type or reference counts, and does not check that the mfn is
755 actually valid. */
756 static int
757 __acquire_grant_for_copy(
758 struct domain *rd, unsigned long gref, int readonly,
759 unsigned long *frame)
760 {
761 grant_entry_t *sha;
762 struct active_grant_entry *act;
763 s16 rc = GNTST_okay;
764 int retries = 0;
765 union grant_combo scombo, prev_scombo, new_scombo;
767 if ( unlikely(gref >= NR_GRANT_ENTRIES) )
768 PIN_FAIL(error_out, GNTST_bad_gntref,
769 "Bad grant reference %ld\n", gref);
771 act = &rd->grant_table->active[gref];
772 sha = &rd->grant_table->shared[gref];
774 spin_lock(&rd->grant_table->lock);
776 /* If already pinned, check the active domid and avoid refcnt overflow. */
777 if ( act->pin &&
778 ((act->domid != current->domain->domain_id) ||
779 (act->pin & 0x80808080U) != 0) )
780 PIN_FAIL(unlock_out, GNTST_general_error,
781 "Bad domain (%d != %d), or risk of counter overflow %08x\n",
782 act->domid, current->domain->domain_id, act->pin);
784 if ( !act->pin ||
785 (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) )
786 {
787 scombo.word = *(u32 *)&sha->flags;
789 for ( ; ; )
790 {
791 /* If not already pinned, check the grant domid and type. */
792 if ( !act->pin &&
793 (((scombo.shorts.flags & GTF_type_mask) !=
794 GTF_permit_access) ||
795 (scombo.shorts.domid != current->domain->domain_id)) )
796 PIN_FAIL(unlock_out, GNTST_general_error,
797 "Bad flags (%x) or dom (%d). (expected dom %d)\n",
798 scombo.shorts.flags, scombo.shorts.domid,
799 current->domain->domain_id);
801 new_scombo = scombo;
802 new_scombo.shorts.flags |= GTF_reading;
804 if ( !readonly )
805 {
806 new_scombo.shorts.flags |= GTF_writing;
807 if ( unlikely(scombo.shorts.flags & GTF_readonly) )
808 PIN_FAIL(unlock_out, GNTST_general_error,
809 "Attempt to write-pin a r/o grant entry.\n");
810 }
812 prev_scombo.word = cmpxchg((u32 *)&sha->flags,
813 scombo.word, new_scombo.word);
814 if ( likely(prev_scombo.word == scombo.word) )
815 break;
817 if ( retries++ == 4 )
818 PIN_FAIL(unlock_out, GNTST_general_error,
819 "Shared grant entry is unstable.\n");
821 scombo = prev_scombo;
822 }
824 if ( !act->pin )
825 {
826 act->domid = scombo.shorts.domid;
827 act->frame = gmfn_to_mfn(rd, sha->frame);
828 }
829 }
831 act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc;
833 *frame = act->frame;
835 unlock_out:
836 spin_unlock(&rd->grant_table->lock);
837 error_out:
838 return rc;
839 }
841 static void
842 __gnttab_copy(
843 struct gnttab_copy *op)
844 {
845 struct domain *sd = NULL, *dd = NULL;
846 unsigned long s_frame, d_frame;
847 char *sp, *dp;
848 s16 rc = GNTST_okay;
849 int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0;
850 int src_is_gref, dest_is_gref;
852 if ( ((op->source.offset + op->len) > PAGE_SIZE) ||
853 ((op->dest.offset + op->len) > PAGE_SIZE) )
854 PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n");
856 src_is_gref = op->flags & GNTCOPY_source_gref;
857 dest_is_gref = op->flags & GNTCOPY_dest_gref;
859 if ( (op->source.domid != DOMID_SELF && !src_is_gref ) ||
860 (op->dest.domid != DOMID_SELF && !dest_is_gref) )
861 PIN_FAIL(error_out, GNTST_permission_denied,
862 "only allow copy-by-mfn for DOMID_SELF.\n");
864 if ( op->source.domid == DOMID_SELF )
865 {
866 sd = current->domain;
867 get_knownalive_domain(sd);
868 }
869 else if ( (sd = find_domain_by_id(op->source.domid)) == NULL )
870 {
871 PIN_FAIL(error_out, GNTST_bad_domain,
872 "couldn't find %d\n", op->source.domid);
873 }
875 if ( op->dest.domid == DOMID_SELF )
876 {
877 dd = current->domain;
878 get_knownalive_domain(dd);
879 }
880 else if ( (dd = find_domain_by_id(op->dest.domid)) == NULL )
881 {
882 PIN_FAIL(error_out, GNTST_bad_domain,
883 "couldn't find %d\n", op->dest.domid);
884 }
886 if ( src_is_gref )
887 {
888 rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame);
889 if ( rc != GNTST_okay )
890 goto error_out;
891 have_s_grant = 1;
892 }
893 else
894 {
895 s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);
896 }
897 if ( unlikely(!mfn_valid(s_frame)) )
898 PIN_FAIL(error_out, GNTST_general_error,
899 "source frame %lx invalid.\n", s_frame);
900 if ( !get_page(mfn_to_page(s_frame), sd) )
901 {
902 if ( !test_bit(_DOMF_dying, &sd->domain_flags) )
903 gdprintk(XENLOG_WARNING, "Could not get src frame %lx\n", s_frame);
904 rc = GNTST_general_error;
905 goto error_out;
906 }
907 have_s_ref = 1;
909 if ( dest_is_gref )
910 {
911 rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame);
912 if ( rc != GNTST_okay )
913 goto error_out;
914 have_d_grant = 1;
915 }
916 else
917 {
918 d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn);
919 }
920 if ( unlikely(!mfn_valid(d_frame)) )
921 PIN_FAIL(error_out, GNTST_general_error,
922 "destination frame %lx invalid.\n", d_frame);
923 if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) )
924 {
925 if ( !test_bit(_DOMF_dying, &dd->domain_flags) )
926 gdprintk(XENLOG_WARNING, "Could not get dst frame %lx\n", d_frame);
927 rc = GNTST_general_error;
928 goto error_out;
929 }
931 sp = map_domain_page(s_frame);
932 dp = map_domain_page(d_frame);
934 memcpy(dp + op->dest.offset, sp + op->source.offset, op->len);
936 unmap_domain_page(dp);
937 unmap_domain_page(sp);
939 gnttab_mark_dirty(dd, d_frame);
941 put_page_and_type(mfn_to_page(d_frame));
942 error_out:
943 if ( have_s_ref )
944 put_page(mfn_to_page(s_frame));
945 if ( have_s_grant )
946 __release_grant_for_copy(sd, op->source.u.ref, 1);
947 if ( have_d_grant )
948 __release_grant_for_copy(dd, op->dest.u.ref, 0);
949 if ( sd )
950 put_domain(sd);
951 if ( dd )
952 put_domain(dd);
953 op->status = rc;
954 }
956 static long
957 gnttab_copy(
958 XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count)
959 {
960 int i;
961 struct gnttab_copy op;
963 for ( i = 0; i < count; i++ )
964 {
965 if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) )
966 return -EFAULT;
967 __gnttab_copy(&op);
968 if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) )
969 return -EFAULT;
970 }
971 return 0;
972 }
974 long
975 do_grant_table_op(
976 unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count)
977 {
978 long rc;
979 struct domain *d = current->domain;
981 if ( count > 512 )
982 return -EINVAL;
984 LOCK_BIGLOCK(d);
986 rc = -EFAULT;
987 switch ( cmd )
988 {
989 case GNTTABOP_map_grant_ref:
990 {
991 XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map =
992 guest_handle_cast(uop, gnttab_map_grant_ref_t);
993 if ( unlikely(!guest_handle_okay(map, count)) )
994 goto out;
995 rc = -EPERM;
996 if ( unlikely(!grant_operation_permitted(d)) )
997 goto out;
998 rc = gnttab_map_grant_ref(map, count);
999 break;
1001 case GNTTABOP_unmap_grant_ref:
1003 XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap =
1004 guest_handle_cast(uop, gnttab_unmap_grant_ref_t);
1005 if ( unlikely(!guest_handle_okay(unmap, count)) )
1006 goto out;
1007 rc = -EPERM;
1008 if ( unlikely(!grant_operation_permitted(d)) )
1009 goto out;
1010 rc = gnttab_unmap_grant_ref(unmap, count);
1011 break;
1013 case GNTTABOP_setup_table:
1015 rc = gnttab_setup_table(
1016 guest_handle_cast(uop, gnttab_setup_table_t), count);
1017 break;
1019 case GNTTABOP_transfer:
1021 XEN_GUEST_HANDLE(gnttab_transfer_t) transfer =
1022 guest_handle_cast(uop, gnttab_transfer_t);
1023 if ( unlikely(!guest_handle_okay(transfer, count)) )
1024 goto out;
1025 rc = -EPERM;
1026 if ( unlikely(!grant_operation_permitted(d)) )
1027 goto out;
1028 rc = gnttab_transfer(transfer, count);
1029 break;
1031 case GNTTABOP_copy:
1033 XEN_GUEST_HANDLE(gnttab_copy_t) copy =
1034 guest_handle_cast(uop, gnttab_copy_t);
1035 if ( unlikely(!guest_handle_okay(copy, count)) )
1036 goto out;
1037 rc = gnttab_copy(copy, count);
1038 break;
1040 default:
1041 rc = -ENOSYS;
1042 break;
1045 out:
1046 UNLOCK_BIGLOCK(d);
1048 return rc;
1051 int
1052 grant_table_create(
1053 struct domain *d)
1055 struct grant_table *t;
1056 int i;
1058 BUG_ON(MAPTRACK_MAX_ENTRIES < NR_GRANT_ENTRIES);
1059 if ( (t = xmalloc(struct grant_table)) == NULL )
1060 goto no_mem;
1062 /* Simple stuff. */
1063 memset(t, 0, sizeof(*t));
1064 spin_lock_init(&t->lock);
1066 /* Active grant table. */
1067 t->active = xmalloc_array(struct active_grant_entry, NR_GRANT_ENTRIES);
1068 if ( t->active == NULL )
1069 goto no_mem;
1070 memset(t->active, 0, sizeof(struct active_grant_entry) * NR_GRANT_ENTRIES);
1072 /* Tracking of mapped foreign frames table */
1073 if ( (t->maptrack = alloc_xenheap_page()) == NULL )
1074 goto no_mem;
1075 t->maptrack_order = 0;
1076 t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping);
1077 memset(t->maptrack, 0, PAGE_SIZE);
1078 for ( i = 0; i < t->maptrack_limit; i++ )
1079 t->maptrack[i].ref = i+1;
1081 /* Shared grant table. */
1082 t->shared = alloc_xenheap_pages(ORDER_GRANT_FRAMES);
1083 if ( t->shared == NULL )
1084 goto no_mem;
1085 memset(t->shared, 0, NR_GRANT_FRAMES * PAGE_SIZE);
1087 for ( i = 0; i < NR_GRANT_FRAMES; i++ )
1088 gnttab_create_shared_page(d, t, i);
1090 /* Okay, install the structure. */
1091 wmb(); /* avoid races with lock-free access to d->grant_table */
1092 d->grant_table = t;
1093 return 0;
1095 no_mem:
1096 if ( t != NULL )
1098 xfree(t->active);
1099 free_xenheap_page(t->maptrack);
1100 xfree(t);
1102 return -ENOMEM;
1105 void
1106 gnttab_release_mappings(
1107 struct domain *d)
1109 struct grant_table *gt = d->grant_table;
1110 struct grant_mapping *map;
1111 grant_ref_t ref;
1112 grant_handle_t handle;
1113 struct domain *rd;
1114 struct active_grant_entry *act;
1115 struct grant_entry *sha;
1117 BUG_ON(!test_bit(_DOMF_dying, &d->domain_flags));
1119 for ( handle = 0; handle < gt->maptrack_limit; handle++ )
1121 map = &gt->maptrack[handle];
1122 if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) )
1123 continue;
1125 ref = map->ref;
1127 gdprintk(XENLOG_INFO, "Grant release (%hu) ref:(%hu) "
1128 "flags:(%x) dom:(%hu)\n",
1129 handle, ref, map->flags, map->domid);
1131 rd = find_domain_by_id(map->domid);
1132 if ( rd == NULL )
1134 /* Nothing to clear up... */
1135 map->flags = 0;
1136 continue;
1139 spin_lock(&rd->grant_table->lock);
1141 act = &rd->grant_table->active[ref];
1142 sha = &rd->grant_table->shared[ref];
1144 if ( map->flags & GNTMAP_readonly )
1146 if ( map->flags & GNTMAP_device_map )
1148 BUG_ON(!(act->pin & GNTPIN_devr_mask));
1149 act->pin -= GNTPIN_devr_inc;
1150 put_page(mfn_to_page(act->frame));
1153 if ( map->flags & GNTMAP_host_map )
1155 BUG_ON(!(act->pin & GNTPIN_hstr_mask));
1156 act->pin -= GNTPIN_hstr_inc;
1157 /* Done implicitly when page tables are destroyed. */
1158 /* put_page(mfn_to_page(act->frame)); */
1161 else
1163 if ( map->flags & GNTMAP_device_map )
1165 BUG_ON(!(act->pin & GNTPIN_devw_mask));
1166 act->pin -= GNTPIN_devw_inc;
1167 put_page_and_type(mfn_to_page(act->frame));
1170 if ( map->flags & GNTMAP_host_map )
1172 BUG_ON(!(act->pin & GNTPIN_hstw_mask));
1173 act->pin -= GNTPIN_hstw_inc;
1174 /* Done implicitly when page tables are destroyed. */
1175 /* put_page_and_type(mfn_to_page(act->frame)); */
1178 if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 )
1179 gnttab_clear_flag(_GTF_writing, &sha->flags);
1182 if ( act->pin == 0 )
1183 gnttab_clear_flag(_GTF_reading, &sha->flags);
1185 spin_unlock(&rd->grant_table->lock);
1187 put_domain(rd);
1189 map->flags = 0;
1194 void
1195 grant_table_destroy(
1196 struct domain *d)
1198 struct grant_table *t = d->grant_table;
1200 if ( t == NULL )
1201 return;
1203 free_xenheap_pages(t->shared, ORDER_GRANT_FRAMES);
1204 free_xenheap_pages(t->maptrack, t->maptrack_order);
1205 xfree(t->active);
1206 xfree(t);
1208 d->grant_table = NULL;
1211 /*
1212 * Local variables:
1213 * mode: C
1214 * c-set-style: "BSD"
1215 * c-basic-offset: 4
1216 * tab-width: 4
1217 * indent-tabs-mode: nil
1218 * End:
1219 */